From 73f3ea23c07870903ee7d37d2dce2b036b1e9edf Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Mon, 14 Apr 2025 17:53:54 +0000 Subject: [PATCH 01/23] refactor: move to cmd dir --- .../readmevalidation}/contributors.go | 0 {scripts/contributors => cmd/readmevalidation}/main.go | 10 ++-------- 2 files changed, 2 insertions(+), 8 deletions(-) rename {scripts/contributors => cmd/readmevalidation}/contributors.go (100%) rename {scripts/contributors => cmd/readmevalidation}/main.go (82%) diff --git a/scripts/contributors/contributors.go b/cmd/readmevalidation/contributors.go similarity index 100% rename from scripts/contributors/contributors.go rename to cmd/readmevalidation/contributors.go diff --git a/scripts/contributors/main.go b/cmd/readmevalidation/main.go similarity index 82% rename from scripts/contributors/main.go rename to cmd/readmevalidation/main.go index 9091318..7c6a2bb 100644 --- a/scripts/contributors/main.go +++ b/cmd/readmevalidation/main.go @@ -18,10 +18,7 @@ func main() { log.Printf("Processing %d README files\n", len(allReadmeFiles)) contributors, err := parseContributorFiles(allReadmeFiles) - log.Printf( - "Processed %d README files as valid contributor profiles", - len(contributors), - ) + log.Printf("Processed %d README files as valid contributor profiles", len(contributors)) if err != nil { log.Panic(err) } @@ -32,8 +29,5 @@ func main() { } log.Println("All relative URLs for READMEs are valid") - log.Printf( - "Processed all READMEs in the %q directory\n", - rootRegistryPath, - ) + log.Printf("Processed all READMEs in the %q directory\n", rootRegistryPath) } From a2abeaee2f2b2555b4acdb452e027c62ddc34fc2 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Mon, 14 Apr 2025 17:58:59 +0000 Subject: [PATCH 02/23] fix: update script references for CI --- .github/workflows/ci.yaml | 4 ++-- .gitignore | 2 +- cmd/readmevalidation/main.go | 11 ++++++----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c27a152..af4ae03 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,6 +17,6 @@ jobs: with: go-version: "1.23.2" - name: Validate contributors - run: go build ./scripts/contributors && ./contributors + run: go build ./cmd/readmevalidation && ./readmevalidation - name: Remove build file artifact - run: rm ./contributors + run: rm ./readmevalidation diff --git a/.gitignore b/.gitignore index 5f109fd..0f945ce 100644 --- a/.gitignore +++ b/.gitignore @@ -136,4 +136,4 @@ dist .pnp.* # Script output -/contributors +/readmevalidation diff --git a/cmd/readmevalidation/main.go b/cmd/readmevalidation/main.go index 7c6a2bb..789797e 100644 --- a/cmd/readmevalidation/main.go +++ b/cmd/readmevalidation/main.go @@ -1,8 +1,9 @@ -// This package is for validating all contributors within the main Registry -// directory. It validates that it has nothing but sub-directories, and that -// each sub-directory has a README.md file. Each of those files must then -// describe a specific contributor. The contents of these files will be parsed -// by the Registry site build step, to be displayed in the Registry site's UI. +// This package is for validating all the README files present in the Registry +// directory. The expectation is that each contributor, module, and template +// will have an associated README containing useful metadata. This metadata must +// be validated for correct structure during CI, because the files themselves +// are parsed and rendered as UI as part of the Registry site build step (the +// Registry site itself lives in a separate repo). package main import ( From 860a633e112316257198edcb40484e289adfcb1e Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Mon, 14 Apr 2025 18:09:43 +0000 Subject: [PATCH 03/23] wip: add support for reading env from CI --- .github/workflows/ci.yaml | 2 ++ cmd/github/githubactions.go | 21 +++++++++++++++++++++ cmd/readmevalidation/main.go | 8 ++++++++ 3 files changed, 31 insertions(+) create mode 100644 cmd/github/githubactions.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index af4ae03..0fd99f5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,6 +9,8 @@ concurrency: jobs: validate-contributors: runs-on: ubuntu-latest + env: + actor: ${{ github.actor }} steps: - name: Check out code uses: actions/checkout@v4 diff --git a/cmd/github/githubactions.go b/cmd/github/githubactions.go new file mode 100644 index 0000000..5d97ab4 --- /dev/null +++ b/cmd/github/githubactions.go @@ -0,0 +1,21 @@ +// Package github provides utilities to make it easier to deal with various +// GitHub APIs +package github + +import ( + "fmt" + "os" +) + +const envActorUsernameKey = "actor" + +// ActionsActor returns the username of the GitHub user who triggered the +// current CI run as part of GitHub Actions.The value must be loaded into the +// env as part of the Github Actions script file, or else the function fails. +func ActionsActor() (string, error) { + username := os.Getenv(envActorUsernameKey) + if username == "" { + return "", fmt.Errorf("value for %q is not in env. Please update the CI script to load the value in during CI", envActorUsernameKey) + } + return username, nil +} diff --git a/cmd/readmevalidation/main.go b/cmd/readmevalidation/main.go index 789797e..c9fe4e9 100644 --- a/cmd/readmevalidation/main.go +++ b/cmd/readmevalidation/main.go @@ -8,9 +8,17 @@ package main import ( "log" + + "coder.com/coder-registry/cmd/github" ) func main() { + username, err := github.ActionsActor() + if err != nil { + log.Panic(err) + } + log.Println("running as %q", username) + log.Println("Starting README validation") allReadmeFiles, err := aggregateContributorReadmeFiles() if err != nil { From ec1b4a72cb7a7a5d408273bd083ce0bd38b2281e Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Mon, 14 Apr 2025 18:13:22 +0000 Subject: [PATCH 04/23] test: see how CI output works --- .github/workflows/ci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0fd99f5..710f6a3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -14,6 +14,8 @@ jobs: steps: - name: Check out code uses: actions/checkout@v4 + - name: Feeling cute. Might (aka should) delete later + run: git branch --show-current - name: Set up Go uses: actions/setup-go@v5 with: From 0a597c23f438be1259bda9ae7778c76c8a6fb972 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Mon, 14 Apr 2025 18:25:24 +0000 Subject: [PATCH 05/23] test: try logging refs --- .github/workflows/ci.yaml | 4 ++-- cmd/github/github.go | 38 +++++++++++++++++++++++++++++++++++++ cmd/github/githubactions.go | 21 -------------------- 3 files changed, 40 insertions(+), 23 deletions(-) create mode 100644 cmd/github/github.go delete mode 100644 cmd/github/githubactions.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 710f6a3..0d3f9ea 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,11 +11,11 @@ jobs: runs-on: ubuntu-latest env: actor: ${{ github.actor }} + base_ref: ${{ github.base_ref }} + head_ref: ${{ github.head_ref }} steps: - name: Check out code uses: actions/checkout@v4 - - name: Feeling cute. Might (aka should) delete later - run: git branch --show-current - name: Set up Go uses: actions/setup-go@v5 with: diff --git a/cmd/github/github.go b/cmd/github/github.go new file mode 100644 index 0000000..7ebb01d --- /dev/null +++ b/cmd/github/github.go @@ -0,0 +1,38 @@ +// Package github provides utilities to make it easier to deal with various +// GitHub APIs +package github + +import ( + "errors" + "fmt" + "os" +) + +const ( + actionsActorKey = "actor" + actionsBaseRefKey = "base_ref" + actionsHeadRefKey = "head_ref" +) + +// ActionsActor returns the username of the GitHub user who triggered the +// current CI run as part of GitHub Actions. The value must be loaded into the +// env as part of the Github Actions YAML file, or else the function fails. +func ActionsActor() (string, error) { + username := os.Getenv(actionsActorKey) + if username == "" { + return "", fmt.Errorf("value for %q is not in env. Please update the CI script to load the value in during CI", actionsActorKey) + } + return username, nil +} + +// ActionsRefs returns the name of the head ref and the base ref for current CI +// run, in that order. Both values must be loaded into the env as part of the +// GitHub Actions YAML file, or else the function fails. +func ActionsRefs() (string, string, error) { + baseRef := os.Getenv(actionsBaseRefKey) + headRef := os.Getenv(actionsHeadRefKey) + fmt.Println("Base ref: ", baseRef) + fmt.Println("Head ref: ", headRef) + + return "", "", errors.New("we ain't ready yet") +} diff --git a/cmd/github/githubactions.go b/cmd/github/githubactions.go deleted file mode 100644 index 5d97ab4..0000000 --- a/cmd/github/githubactions.go +++ /dev/null @@ -1,21 +0,0 @@ -// Package github provides utilities to make it easier to deal with various -// GitHub APIs -package github - -import ( - "fmt" - "os" -) - -const envActorUsernameKey = "actor" - -// ActionsActor returns the username of the GitHub user who triggered the -// current CI run as part of GitHub Actions.The value must be loaded into the -// env as part of the Github Actions script file, or else the function fails. -func ActionsActor() (string, error) { - username := os.Getenv(envActorUsernameKey) - if username == "" { - return "", fmt.Errorf("value for %q is not in env. Please update the CI script to load the value in during CI", envActorUsernameKey) - } - return username, nil -} From d2c5f8d3bd122d6d0865b0cb94359b8ce7e6bf1c Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Mon, 14 Apr 2025 18:27:10 +0000 Subject: [PATCH 06/23] fix: actually add the test calls --- cmd/readmevalidation/main.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/readmevalidation/main.go b/cmd/readmevalidation/main.go index c9fe4e9..07cbf95 100644 --- a/cmd/readmevalidation/main.go +++ b/cmd/readmevalidation/main.go @@ -17,7 +17,11 @@ func main() { if err != nil { log.Panic(err) } - log.Println("running as %q", username) + log.Printf("running as %q\n", username) + _, _, err = github.ActionsRefs() + if err != nil { + log.Panic(err) + } log.Println("Starting README validation") allReadmeFiles, err := aggregateContributorReadmeFiles() From 25d301c654a5e37ec81ac8381eb16cbc71af1d6b Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Mon, 14 Apr 2025 20:40:25 +0000 Subject: [PATCH 07/23] wip: commit progress --- .env_example | 5 ++ .github/workflows/ci.yaml | 8 ++- cmd/github/github.go | 132 +++++++++++++++++++++++++++++++++-- cmd/readmevalidation/main.go | 16 ++++- go.mod | 2 + go.sum | 2 + 6 files changed, 153 insertions(+), 12 deletions(-) create mode 100644 .env_example diff --git a/.env_example b/.env_example new file mode 100644 index 0000000..267b203 --- /dev/null +++ b/.env_example @@ -0,0 +1,5 @@ +ACTOR= +BASE_REF= +HEAD_REF= +GITHUB_API_URL= +GITHUB_API_TOKEN= \ No newline at end of file diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0d3f9ea..889c46c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,9 +10,11 @@ jobs: validate-contributors: runs-on: ubuntu-latest env: - actor: ${{ github.actor }} - base_ref: ${{ github.base_ref }} - head_ref: ${{ github.head_ref }} + ACTOR: ${{ github.actor }} + BASE_REF: ${{ github.base_ref }} + HEAD_REF: ${{ github.head_ref }} + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Check out code uses: actions/checkout@v4 diff --git a/cmd/github/github.go b/cmd/github/github.go index 7ebb01d..d5f6784 100644 --- a/cmd/github/github.go +++ b/cmd/github/github.go @@ -3,15 +3,28 @@ package github import ( + "encoding/json" "errors" "fmt" + "io" + "log" + "net/http" "os" + "strings" + "time" ) +const defaultGithubAPIRoute = "https://api.github.com/" + const ( - actionsActorKey = "actor" - actionsBaseRefKey = "base_ref" - actionsHeadRefKey = "head_ref" + actionsActorKey = "ACTOR" + actionsBaseRefKey = "BASE_REF" + actionsHeadRefKey = "HEAD_REF" +) + +const ( + githubAPIURLKey = "GITHUB_API_URL" + githubAPITokenKey = "GITHUB_API_TOKEN" ) // ActionsActor returns the username of the GitHub user who triggered the @@ -20,7 +33,7 @@ const ( func ActionsActor() (string, error) { username := os.Getenv(actionsActorKey) if username == "" { - return "", fmt.Errorf("value for %q is not in env. Please update the CI script to load the value in during CI", actionsActorKey) + return "", fmt.Errorf("value for %q is not in env. If running from CI, please add value via ci.yaml file", actionsActorKey) } return username, nil } @@ -31,8 +44,113 @@ func ActionsActor() (string, error) { func ActionsRefs() (string, string, error) { baseRef := os.Getenv(actionsBaseRefKey) headRef := os.Getenv(actionsHeadRefKey) - fmt.Println("Base ref: ", baseRef) - fmt.Println("Head ref: ", headRef) - return "", "", errors.New("we ain't ready yet") + if baseRef == "" && headRef == "" { + return "", "", fmt.Errorf("values for %q and %q are not in env. If running from CI, please add values via ci.yaml file", actionsHeadRefKey, actionsBaseRefKey) + } else if headRef == "" { + return "", "", fmt.Errorf("value for %q is not in env. If running from CI, please add value via ci.yaml file", actionsHeadRefKey) + } else if baseRef == "" { + return "", "", fmt.Errorf("value for %q is not in env. If running from CI, please add value via ci.yaml file", actionsBaseRefKey) + } + + return headRef, baseRef, nil +} + +// CoderEmployees represents all members of the Coder GitHub organization. This +// value should not be instantiated from outside the package, and should instead +// be created via one of the package's exported functions. +type CoderEmployees struct { + // Have map defined as private field to make sure that it can't ever be + // mutated from an outside package + _employees map[string]struct{} +} + +// IsEmployee takes a GitHub username and indicates whether the matching user is +// a member of the Coder organization +func (ce *CoderEmployees) IsEmployee(username string) bool { + if ce._employees == nil { + return false + } + + _, ok := ce._employees[username] + return ok +} + +// TotalEmployees returns the number of members in the Coder organization +func (ce *CoderEmployees) TotalEmployees() int { + return len(ce._employees) +} + +type ghOrganizationMember struct { + Login string `json:"login"` +} + +type ghRateLimitedRes struct { + Message string `json:"message"` +} + +func parseResponse[V any](b []byte) (V, error) { + var want V + var rateLimitedRes ghRateLimitedRes + + if err := json.Unmarshal(b, &rateLimitedRes); err != nil { + return want, err + } + if isRateLimited := strings.Contains(rateLimitedRes.Message, "API rate limit exceeded for "); isRateLimited { + return want, errors.New("request was rate-limited") + } + if err := json.Unmarshal(b, &want); err != nil { + return want, err + } + + return want, nil +} + +// CoderEmployeeUsernames requests from the GitHub API the list of all usernames +// of people who are employees of Coder. +func CoderEmployeeUsernames() (CoderEmployees, error) { + apiURL := os.Getenv(githubAPIURLKey) + if apiURL == "" { + log.Printf("API URL not set via env key %q. Defaulting to %q\n", githubAPIURLKey, defaultGithubAPIRoute) + apiURL = defaultGithubAPIRoute + } + token := os.Getenv(githubAPITokenKey) + if token == "" { + log.Printf("API token not set via env key %q. All requests will be non-authenticated and subject to more aggressive rate limiting", githubAPITokenKey) + } + + req, err := http.NewRequest("GET", apiURL+"/orgs/coder/members", nil) + if err != nil { + return CoderEmployees{}, fmt.Errorf("coder employee names: %v", err) + } + if token != "" { + req.Header.Add("Authorization", "Bearer "+token) + } + + client := http.Client{Timeout: 5 * time.Second} + res, err := client.Do(req) + if err != nil { + return CoderEmployees{}, fmt.Errorf("coder employee names: %v", err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return CoderEmployees{}, fmt.Errorf("coder employee names: got back status code %d", res.StatusCode) + } + + b, err := io.ReadAll(res.Body) + if err != nil { + return CoderEmployees{}, fmt.Errorf("coder employee names: %v", err) + } + rawMembers, err := parseResponse[[]ghOrganizationMember](b) + if err != nil { + return CoderEmployees{}, fmt.Errorf("coder employee names: %v", err) + } + + employeesSet := map[string]struct{}{} + for _, m := range rawMembers { + employeesSet[m.Login] = struct{}{} + } + return CoderEmployees{ + _employees: employeesSet, + }, nil } diff --git a/cmd/readmevalidation/main.go b/cmd/readmevalidation/main.go index 07cbf95..8fd657c 100644 --- a/cmd/readmevalidation/main.go +++ b/cmd/readmevalidation/main.go @@ -10,18 +10,30 @@ import ( "log" "coder.com/coder-registry/cmd/github" + "github.com/joho/godotenv" ) func main() { + err := godotenv.Load() + if err != nil { + log.Panic(err) + } username, err := github.ActionsActor() if err != nil { log.Panic(err) } - log.Printf("running as %q\n", username) - _, _, err = github.ActionsRefs() + log.Printf("Running validation for user %q", username) + headRef, baseRef, err := github.ActionsRefs() + if err != nil { + log.Panic(err) + } + log.Printf("Using branches %q and %q for validation comparison", headRef, baseRef) + + employees, err := github.CoderEmployeeUsernames() if err != nil { log.Panic(err) } + log.Printf("got back %d employees\n", employees.TotalEmployees()) log.Println("Starting README validation") allReadmeFiles, err := aggregateContributorReadmeFiles() diff --git a/go.mod b/go.mod index e407422..61231b9 100644 --- a/go.mod +++ b/go.mod @@ -3,3 +3,5 @@ module coder.com/coder-registry go 1.23.2 require gopkg.in/yaml.v3 v3.0.1 + +require github.com/joho/godotenv v1.5.1 // indirect diff --git a/go.sum b/go.sum index a62c313..e536c30 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From 9f035798d13e0c394a4ece66e1990c3012425237 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 15 Apr 2025 14:32:38 +0000 Subject: [PATCH 08/23] wip: commit progress --- .env.example | 23 ++++ .env_example | 5 - cmd/github/github.go | 208 ++++++++++++++++++++--------------- cmd/readmevalidation/main.go | 30 ++++- 4 files changed, 169 insertions(+), 97 deletions(-) create mode 100644 .env.example delete mode 100644 .env_example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..6bf034b --- /dev/null +++ b/.env.example @@ -0,0 +1,23 @@ +# This should be the value of the GitHub Actions actor who triggered a run. The +# CI script will inject this value from the GitHub Actions context to verify +# whether changing certain README fields is allowed. In local development, you +# can set this to your GitHub username. +CI_ACTOR= + +# This is the Git ref that you want to merge into the main branch. In local +# development, this should be set to the value of the branch you're working from +CI_BASE_REF= + +# This is the configurable base URL for accessing the GitHub REST API. This +# value will be injected by the CI script's Actions context, but if the value is +# not defined (either in CI or when running locally), "https://api.github.com/" +# will be used as a fallback. +GITHUB_API_URL= + +# This is the API token for the user that will be used to authenticate calls to +# the GitHub API. In CI, the value will be loaded with a token belonging to a +# Coder Registry admin to verify whether modifying certain README fields is +# allowed. In local development, you can set a token with the read:org +# permission. If the loaded token does not belong to a Coder employee, certain +# README verification steps will be skipped. +GITHUB_API_TOKEN= diff --git a/.env_example b/.env_example deleted file mode 100644 index 267b203..0000000 --- a/.env_example +++ /dev/null @@ -1,5 +0,0 @@ -ACTOR= -BASE_REF= -HEAD_REF= -GITHUB_API_URL= -GITHUB_API_TOKEN= \ No newline at end of file diff --git a/cmd/github/github.go b/cmd/github/github.go index d5f6784..68b40f1 100644 --- a/cmd/github/github.go +++ b/cmd/github/github.go @@ -10,16 +10,14 @@ import ( "log" "net/http" "os" - "strings" "time" ) -const defaultGithubAPIRoute = "https://api.github.com/" +const defaultGithubAPIBaseRoute = "https://api.github.com/" const ( - actionsActorKey = "ACTOR" - actionsBaseRefKey = "BASE_REF" - actionsHeadRefKey = "HEAD_REF" + actionsActorKey = "CI_ACTOR" + actionsBaseRefKey = "CI_BASE_REF" ) const ( @@ -38,119 +36,157 @@ func ActionsActor() (string, error) { return username, nil } -// ActionsRefs returns the name of the head ref and the base ref for current CI -// run, in that order. Both values must be loaded into the env as part of the -// GitHub Actions YAML file, or else the function fails. -func ActionsRefs() (string, string, error) { +// BaseRef returns the name of the base ref for the Git branch that will be +// merged into the main branch. +func BaseRef() (string, error) { baseRef := os.Getenv(actionsBaseRefKey) - headRef := os.Getenv(actionsHeadRefKey) - - if baseRef == "" && headRef == "" { - return "", "", fmt.Errorf("values for %q and %q are not in env. If running from CI, please add values via ci.yaml file", actionsHeadRefKey, actionsBaseRefKey) - } else if headRef == "" { - return "", "", fmt.Errorf("value for %q is not in env. If running from CI, please add value via ci.yaml file", actionsHeadRefKey) - } else if baseRef == "" { - return "", "", fmt.Errorf("value for %q is not in env. If running from CI, please add value via ci.yaml file", actionsBaseRefKey) + if baseRef == "" { + return "", fmt.Errorf("value for %q is not in env. If running from CI, please add value via ci.yaml file", actionsBaseRefKey) } - return headRef, baseRef, nil + return baseRef, nil } -// CoderEmployees represents all members of the Coder GitHub organization. This -// value should not be instantiated from outside the package, and should instead -// be created via one of the package's exported functions. -type CoderEmployees struct { - // Have map defined as private field to make sure that it can't ever be - // mutated from an outside package - _employees map[string]struct{} +// Client is a reusable REST client for making requests to the GitHub API. +// It should be instantiated via NewGithubClient +type Client struct { + baseURL string + token string + httpClient http.Client } -// IsEmployee takes a GitHub username and indicates whether the matching user is -// a member of the Coder organization -func (ce *CoderEmployees) IsEmployee(username string) bool { - if ce._employees == nil { - return false +// NewClient instantiates a GitHub client +func NewClient() (*Client, error) { + // Considered letting the user continue on with no token and more aggressive + // rate-limiting, but from experimentation, the non-authenticated experience + // hit the rate limits really quickly, and had a lot of restrictions + apiToken := os.Getenv(githubAPITokenKey) + if apiToken == "" { + return nil, fmt.Errorf("missing env variable %q", githubAPITokenKey) } - _, ok := ce._employees[username] - return ok -} + baseURL := os.Getenv(githubAPIURLKey) + if baseURL == "" { + log.Printf("env variable %q is not defined. Falling back to %q\n", githubAPIURLKey, defaultGithubAPIBaseRoute) + baseURL = defaultGithubAPIBaseRoute + } -// TotalEmployees returns the number of members in the Coder organization -func (ce *CoderEmployees) TotalEmployees() int { - return len(ce._employees) + return &Client{ + baseURL: baseURL, + token: apiToken, + httpClient: http.Client{Timeout: 10 * time.Second}, + }, nil } -type ghOrganizationMember struct { +// User represents a truncated version of the API response from Github's /user +// endpoint. +type User struct { Login string `json:"login"` } -type ghRateLimitedRes struct { - Message string `json:"message"` -} - -func parseResponse[V any](b []byte) (V, error) { - var want V - var rateLimitedRes ghRateLimitedRes - - if err := json.Unmarshal(b, &rateLimitedRes); err != nil { - return want, err - } - if isRateLimited := strings.Contains(rateLimitedRes.Message, "API rate limit exceeded for "); isRateLimited { - return want, errors.New("request was rate-limited") +// GetUserFromToken returns the user associated with the loaded API token +func (gc *Client) GetUserFromToken() (User, error) { + req, err := http.NewRequest("GET", gc.baseURL+"user", nil) + if err != nil { + return User{}, err } - if err := json.Unmarshal(b, &want); err != nil { - return want, err + if gc.token != "" { + req.Header.Add("Authorization", "Bearer "+gc.token) } - return want, nil -} + res, err := gc.httpClient.Do(req) + if err != nil { + return User{}, err + } + defer res.Body.Close() -// CoderEmployeeUsernames requests from the GitHub API the list of all usernames -// of people who are employees of Coder. -func CoderEmployeeUsernames() (CoderEmployees, error) { - apiURL := os.Getenv(githubAPIURLKey) - if apiURL == "" { - log.Printf("API URL not set via env key %q. Defaulting to %q\n", githubAPIURLKey, defaultGithubAPIRoute) - apiURL = defaultGithubAPIRoute + if res.StatusCode == http.StatusUnauthorized { + return User{}, errors.New("request is not authorized") } - token := os.Getenv(githubAPITokenKey) - if token == "" { - log.Printf("API token not set via env key %q. All requests will be non-authenticated and subject to more aggressive rate limiting", githubAPITokenKey) + if res.StatusCode == http.StatusForbidden { + return User{}, errors.New("request is forbidden") } - req, err := http.NewRequest("GET", apiURL+"/orgs/coder/members", nil) + b, err := io.ReadAll(res.Body) if err != nil { - return CoderEmployees{}, fmt.Errorf("coder employee names: %v", err) - } - if token != "" { - req.Header.Add("Authorization", "Bearer "+token) + return User{}, err } - client := http.Client{Timeout: 5 * time.Second} - res, err := client.Do(req) - if err != nil { - return CoderEmployees{}, fmt.Errorf("coder employee names: %v", err) + user := User{} + if err := json.Unmarshal(b, &user); err != nil { + return User{}, err } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return CoderEmployees{}, fmt.Errorf("coder employee names: got back status code %d", res.StatusCode) + return user, nil +} + +// OrgStatus indicates whether a GitHub user is a member of a given organization +type OrgStatus int + +var _ fmt.Stringer = OrgStatus(0) + +const ( + // OrgStatusIndeterminate indicates when a user's organization status + // could not be determined. It is the zero value of the OrgStatus type, and + // any users with this value should be treated as completely untrusted + OrgStatusIndeterminate = iota + // OrgStatusNonMember indicates when a user is definitely NOT part of an + // organization + OrgStatusNonMember + // OrgStatusMember indicates when a user is a member of a Github + // organization + OrgStatusMember +) + +func (s OrgStatus) String() string { + switch s { + case OrgStatusMember: + return "Member" + case OrgStatusNonMember: + return "Non-member" + default: + return "Indeterminate" } +} - b, err := io.ReadAll(res.Body) +// GetUserOrgStatus takes a GitHub username, and checks the GitHub API to see +// whether that member is part of the Coder organization +func (gc *Client) GetUserOrgStatus(org string, username string) (OrgStatus, error) { + // This API endpoint is really annoying, because it's able to produce false + // negatives. Any user can be a public member of Coder, a private member of + // Coder, or a non-member. + // + // So if the function returns status 200, you can always trust that. But if + // it returns any 400 code, that could indicate a few things: + // 1. The user being checked is not part of the organization, but the user + // associated with the token is. + // 2. The user being checked is a member of the organization, but their + // status is private, and the token being used to check belongs to a user + // who is not part of the Coder organization. + // 3. Neither the user being checked nor the user associated with the token + // are members of the organization + // + // The best option is to make sure that the token being used belongs to a + // member of the Coder organization + req, err := http.NewRequest("GET", fmt.Sprintf("%sorgs/%s/%s", gc.baseURL, org, username), nil) if err != nil { - return CoderEmployees{}, fmt.Errorf("coder employee names: %v", err) + return OrgStatusIndeterminate, err + } + if gc.token != "" { + req.Header.Add("Authorization", "Bearer "+gc.token) } - rawMembers, err := parseResponse[[]ghOrganizationMember](b) + + res, err := gc.httpClient.Do(req) if err != nil { - return CoderEmployees{}, fmt.Errorf("coder employee names: %v", err) + return OrgStatusIndeterminate, err } + defer res.Body.Close() - employeesSet := map[string]struct{}{} - for _, m := range rawMembers { - employeesSet[m.Login] = struct{}{} + switch res.StatusCode { + case http.StatusNoContent: + return OrgStatusMember, nil + case http.StatusNotFound: + return OrgStatusNonMember, nil + default: + return OrgStatusIndeterminate, nil } - return CoderEmployees{ - _employees: employeesSet, - }, nil } diff --git a/cmd/readmevalidation/main.go b/cmd/readmevalidation/main.go index 8fd657c..9a5aaf9 100644 --- a/cmd/readmevalidation/main.go +++ b/cmd/readmevalidation/main.go @@ -7,6 +7,7 @@ package main import ( + "fmt" "log" "coder.com/coder-registry/cmd/github" @@ -14,26 +15,43 @@ import ( ) func main() { + log.Println("Beginning README file validation") err := godotenv.Load() if err != nil { log.Panic(err) } - username, err := github.ActionsActor() + actorUsername, err := github.ActionsActor() if err != nil { log.Panic(err) } - log.Printf("Running validation for user %q", username) - headRef, baseRef, err := github.ActionsRefs() + baseRef, err := github.BaseRef() if err != nil { log.Panic(err) } - log.Printf("Using branches %q and %q for validation comparison", headRef, baseRef) + log.Printf("Using branch %q for validation comparison", baseRef) - employees, err := github.CoderEmployeeUsernames() + log.Printf("Using GitHub API to determine what fields can be set by user %q\n", actorUsername) + client, err := github.NewClient() if err != nil { log.Panic(err) } - log.Printf("got back %d employees\n", employees.TotalEmployees()) + tokenUser, err := client.GetUserFromToken() + if err != nil { + log.Panic(err) + } + tokenUserStatus, err := client.GetUserOrgStatus("coder", tokenUser.Login) + if err != nil { + log.Panic(err) + } + var actorOrgStatus github.OrgStatus + if tokenUserStatus == github.OrgStatusMember { + actorOrgStatus, err = client.GetUserOrgStatus("coder", actorUsername) + if err != nil { + log.Panic(err) + } + } + + fmt.Printf("actor %q is %s\n", actorUsername, actorOrgStatus.String()) log.Println("Starting README validation") allReadmeFiles, err := aggregateContributorReadmeFiles() From 3fa316dc3785d5352ee7d0e5dbeb207ae0b03f64 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 15 Apr 2025 14:54:30 +0000 Subject: [PATCH 09/23] wip: get basic GH API call stuff working --- cmd/github/github.go | 29 +++++++++++++++------------- cmd/readmevalidation/contributors.go | 2 +- cmd/readmevalidation/main.go | 12 +++++++----- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/cmd/github/github.go b/cmd/github/github.go index 68b40f1..00b3f3a 100644 --- a/cmd/github/github.go +++ b/cmd/github/github.go @@ -149,25 +149,28 @@ func (s OrgStatus) String() string { } // GetUserOrgStatus takes a GitHub username, and checks the GitHub API to see -// whether that member is part of the Coder organization -func (gc *Client) GetUserOrgStatus(org string, username string) (OrgStatus, error) { +// whether that member is part of the provided organization +func (gc *Client) GetUserOrgStatus(orgName string, username string) (OrgStatus, error) { // This API endpoint is really annoying, because it's able to produce false - // negatives. Any user can be a public member of Coder, a private member of - // Coder, or a non-member. + // negatives. Any user can be: + // 1. A public member of an organization + // 2. A private member of an organization + // 3. Not a member of an organization // // So if the function returns status 200, you can always trust that. But if // it returns any 400 code, that could indicate a few things: - // 1. The user being checked is not part of the organization, but the user - // associated with the token is. - // 2. The user being checked is a member of the organization, but their - // status is private, and the token being used to check belongs to a user - // who is not part of the Coder organization. + // 1. The user associated with the token is a member of the organization, + // and the user being checked is not. + // 2. The user associated with the token is NOT a member of the + // organization, and the member being checked is a private member. The + // token user will have no way to view the private member's status. // 3. Neither the user being checked nor the user associated with the token - // are members of the organization + // are members of the organization. // - // The best option is to make sure that the token being used belongs to a - // member of the Coder organization - req, err := http.NewRequest("GET", fmt.Sprintf("%sorgs/%s/%s", gc.baseURL, org, username), nil) + // The best option to avoid false positives is to make sure that the token + // being used belongs to a member of the organization being checked. + url := fmt.Sprintf("%sorgs/%s/members/%s", gc.baseURL, orgName, username) + req, err := http.NewRequest("GET", url, nil) if err != nil { return OrgStatusIndeterminate, err } diff --git a/cmd/readmevalidation/contributors.go b/cmd/readmevalidation/contributors.go index 02823f2..0d081bb 100644 --- a/cmd/readmevalidation/contributors.go +++ b/cmd/readmevalidation/contributors.go @@ -405,7 +405,7 @@ func aggregateContributorReadmeFiles() ([]readme, error) { return allReadmeFiles, nil } -func validateRelativeUrls( +func validateContributorRelativeUrls( contributors map[string]contributorProfile, ) error { // This function only validates relative avatar URLs for now, but it can be diff --git a/cmd/readmevalidation/main.go b/cmd/readmevalidation/main.go index 9a5aaf9..f14ed12 100644 --- a/cmd/readmevalidation/main.go +++ b/cmd/readmevalidation/main.go @@ -15,6 +15,7 @@ import ( ) func main() { + // Do basic setup log.Println("Beginning README file validation") err := godotenv.Load() if err != nil { @@ -30,6 +31,8 @@ func main() { } log.Printf("Using branch %q for validation comparison", baseRef) + // Retrieve data necessary from the GitHub API to help determine whether + // certain field changes are allowed log.Printf("Using GitHub API to determine what fields can be set by user %q\n", actorUsername) client, err := github.NewClient() if err != nil { @@ -49,28 +52,27 @@ func main() { if err != nil { log.Panic(err) } + } else { + log.Println("Provided API token does not belong to a Coder employee. Some README validation steps will be skipped compared to when they run in CI.") } - fmt.Printf("actor %q is %s\n", actorUsername, actorOrgStatus.String()) log.Println("Starting README validation") + allReadmeFiles, err := aggregateContributorReadmeFiles() if err != nil { log.Panic(err) } - log.Printf("Processing %d README files\n", len(allReadmeFiles)) contributors, err := parseContributorFiles(allReadmeFiles) log.Printf("Processed %d README files as valid contributor profiles", len(contributors)) if err != nil { log.Panic(err) } - - err = validateRelativeUrls(contributors) + err = validateContributorRelativeUrls(contributors) if err != nil { log.Panic(err) } log.Println("All relative URLs for READMEs are valid") - log.Printf("Processed all READMEs in the %q directory\n", rootRegistryPath) } From 6e5d96087127024528f3f3f4481de378682a0c89 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 15 Apr 2025 15:06:16 +0000 Subject: [PATCH 10/23] refactor: split README logic into separate file --- cmd/readmevalidation/contributors.go | 49 +---------- cmd/readmevalidation/readmes.go | 127 +++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 47 deletions(-) create mode 100644 cmd/readmevalidation/readmes.go diff --git a/cmd/readmevalidation/contributors.go b/cmd/readmevalidation/contributors.go index 0d081bb..fbcc338 100644 --- a/cmd/readmevalidation/contributors.go +++ b/cmd/readmevalidation/contributors.go @@ -1,7 +1,6 @@ package main import ( - "bufio" "errors" "fmt" "net/url" @@ -13,17 +12,7 @@ import ( "gopkg.in/yaml.v3" ) -const rootRegistryPath = "./registry" - -var ( - validContributorStatuses = []string{"official", "partner", "community"} - supportedAvatarFileFormats = []string{".png", ".jpeg", ".jpg", ".gif", ".svg"} -) - -type readme struct { - filePath string - rawText string -} +var validContributorStatuses = []string{"official", "partner", "community"} type contributorProfileFrontmatter struct { DisplayName string `yaml:"display_name"` @@ -65,40 +54,6 @@ func (vpe validationPhaseError) Error() string { return msg } -func extractFrontmatter(readmeText string) (string, error) { - if readmeText == "" { - return "", errors.New("README is empty") - } - - const fence = "---" - fm := "" - fenceCount := 0 - lineScanner := bufio.NewScanner( - strings.NewReader(strings.TrimSpace(readmeText)), - ) - for lineScanner.Scan() { - nextLine := lineScanner.Text() - if fenceCount == 0 && nextLine != fence { - return "", errors.New("README does not start with frontmatter fence") - } - - if nextLine != fence { - fm += nextLine + "\n" - continue - } - - fenceCount++ - if fenceCount >= 2 { - break - } - } - - if fenceCount == 1 { - return "", errors.New("README does not have two sets of frontmatter fences") - } - return fm, nil -} - func validateContributorGithubUsername(githubUsername string) error { if githubUsername == "" { return errors.New("missing GitHub username") @@ -297,7 +252,7 @@ func validateContributorYaml(yml contributorProfile) []error { } func parseContributorProfile(rm readme) (contributorProfile, error) { - fm, err := extractFrontmatter(rm.rawText) + fm, _, err := separateFrontmatter(rm.rawText) if err != nil { return contributorProfile{}, fmt.Errorf("%q: failed to parse frontmatter: %v", rm.filePath, err) } diff --git a/cmd/readmevalidation/readmes.go b/cmd/readmevalidation/readmes.go new file mode 100644 index 0000000..e6aff86 --- /dev/null +++ b/cmd/readmevalidation/readmes.go @@ -0,0 +1,127 @@ +package main + +import ( + "bufio" + "errors" + "fmt" + "strings" +) + +const rootRegistryPath = "./registry" + +var supportedAvatarFileFormats = []string{".png", ".jpeg", ".jpg", ".gif", ".svg"} + +// Readme represents a single README file within the repo (usually within the +// "/registry" directory). +type readme struct { + filePath string + rawText string +} + +// separateFrontmatter attempts to separate a README file's frontmatter content +// from the main README body, returning both values in that order. It does not +// validate whether the structure of the frontmatter is valid (i.e., that it's +// structured as YAML). +func separateFrontmatter(readmeText string) (string, string, error) { + if readmeText == "" { + return "", "", errors.New("README is empty") + } + + const fence = "---" + fm := "" + body := "" + fenceCount := 0 + lineScanner := bufio.NewScanner( + strings.NewReader(strings.TrimSpace(readmeText)), + ) + for lineScanner.Scan() { + nextLine := lineScanner.Text() + if fenceCount < 2 && nextLine == fence { + fenceCount++ + continue + } + // Break early if the very first line wasn't a fence, because then we + // know for certain that the README has problems + if fenceCount == 0 { + break + } + + // It should be safe to trim each line of the frontmatter on a per-line + // basis, because there shouldn't be any extra meaning attached to the + // indentation. The same does NOT apply to the README; best we can do is + // gather all the lines, and then trim around it + if inReadmeBody := fenceCount >= 2; inReadmeBody { + body += nextLine + "\n" + } else { + fm += strings.TrimSpace(nextLine) + "\n" + } + } + if fenceCount < 2 { + return "", "", errors.New("README does not have two sets of frontmatter fences") + } + if fm == "" { + return "", "", errors.New("readme has frontmatter fences but no frontmatter content") + } + + return fm, strings.TrimSpace(body), nil +} + +// validationPhase represents a specific phase during README validation. It is +// expected that each phase is discrete, and errors during one will prevent a +// future phase from starting. +type validationPhase int + +const ( + // validationPhaseFilesystemRead indicates when a README file is being read + // from the file system + validationPhaseFilesystemRead validationPhase = iota + + // validationPhaseReadmeParsing indicates when a README's frontmatter is being + // parsed as YAML. This phase does not include YAML validation. + validationPhaseReadmeParsing + + // validationPhaseReadmeValidation indicates when a README's frontmatter is + // being validated as proper YAML with expected keys. + validationPhaseReadmeValidation + + // validationPhaseAssetCrossReference indicates when a README's frontmatter + // is having all its relative URLs be validated for whether they point to + // valid resources. + validationPhaseAssetCrossReference +) + +func (p validationPhase) String() string { + switch p { + case validationPhaseFilesystemRead: + return "Filesystem reading" + case validationPhaseReadmeParsing: + return "README parsing" + case validationPhaseReadmeValidation: + return "README validation" + case validationPhaseAssetCrossReference: + return "Cross-referencing asset references" + default: + return "Unknown validation phase" + } +} + +var _ error = ValidationPhaseError{} + +// ValidationPhaseError represents an error that occurred during a specific +// phase of README validation. It should be used to collect ALL validation +// errors that happened during a specific phase, rather than the first one +// encountered. +type ValidationPhaseError struct { + phase validationPhase + errors []error +} + +func (vpe ValidationPhaseError) Error() string { + msg := fmt.Sprintf("Error during %q phase of README validation:", vpe.phase.String()) + for _, e := range vpe.errors { + msg += fmt.Sprintf("\n- %v", e) + } + msg += "\n" + + return msg +} From 94ca584b9eafdc0f6f9ae296230e567db7e7e648 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 15 Apr 2025 15:12:03 +0000 Subject: [PATCH 11/23] refactor: move where validation phase is defined --- cmd/readmevalidation/contributors.go | 33 +++++----------------------- cmd/readmevalidation/readmes.go | 8 +++---- 2 files changed, 10 insertions(+), 31 deletions(-) diff --git a/cmd/readmevalidation/contributors.go b/cmd/readmevalidation/contributors.go index fbcc338..8dd6f81 100644 --- a/cmd/readmevalidation/contributors.go +++ b/cmd/readmevalidation/contributors.go @@ -33,27 +33,6 @@ type contributorProfile struct { filePath string } -var _ error = validationPhaseError{} - -type validationPhaseError struct { - phase string - errors []error -} - -func (vpe validationPhaseError) Error() string { - validationStrs := []string{} - for _, e := range vpe.errors { - validationStrs = append(validationStrs, fmt.Sprintf("- %v", e)) - } - slices.Sort(validationStrs) - - msg := fmt.Sprintf("Error during %q phase of README validation:", vpe.phase) - msg += strings.Join(validationStrs, "\n") - msg += "\n" - - return msg -} - func validateContributorGithubUsername(githubUsername string) error { if githubUsername == "" { return errors.New("missing GitHub username") @@ -219,7 +198,7 @@ func addFilePathToError(filePath string, err error) error { return fmt.Errorf("%q: %v", filePath, err) } -func validateContributorYaml(yml contributorProfile) []error { +func validateContributorProfile(yml contributorProfile) []error { allProblems := []error{} if err := validateContributorGithubUsername(yml.frontmatter.GithubUsername); err != nil { @@ -286,7 +265,7 @@ func parseContributorFiles(readmeEntries []readme) (map[string]contributorProfil } if len(yamlParsingErrors) != 0 { return nil, validationPhaseError{ - phase: "YAML parsing", + phase: validationPhaseReadmeParsing, errors: yamlParsingErrors, } } @@ -294,7 +273,7 @@ func parseContributorFiles(readmeEntries []readme) (map[string]contributorProfil employeeGithubGroups := map[string][]string{} yamlValidationErrors := []error{} for _, p := range profilesByUsername { - errors := validateContributorYaml(p) + errors := validateContributorProfile(p) if len(errors) > 0 { yamlValidationErrors = append(yamlValidationErrors, errors...) continue @@ -315,7 +294,7 @@ func parseContributorFiles(readmeEntries []readme) (map[string]contributorProfil } if len(yamlValidationErrors) != 0 { return nil, validationPhaseError{ - phase: "Raw YAML Validation", + phase: validationPhaseReadmeValidation, errors: yamlValidationErrors, } } @@ -352,7 +331,7 @@ func aggregateContributorReadmeFiles() ([]readme, error) { if len(problems) != 0 { return nil, validationPhaseError{ - phase: "FileSystem reading", + phase: validationPhaseFilesystemRead, errors: problems, } } @@ -395,7 +374,7 @@ func validateContributorRelativeUrls( return nil } return validationPhaseError{ - phase: "Relative URL validation", + phase: validationPhaseAssetCrossReference, errors: problems, } } diff --git a/cmd/readmevalidation/readmes.go b/cmd/readmevalidation/readmes.go index e6aff86..47bb196 100644 --- a/cmd/readmevalidation/readmes.go +++ b/cmd/readmevalidation/readmes.go @@ -105,18 +105,18 @@ func (p validationPhase) String() string { } } -var _ error = ValidationPhaseError{} +var _ error = validationPhaseError{} -// ValidationPhaseError represents an error that occurred during a specific +// validationPhaseError represents an error that occurred during a specific // phase of README validation. It should be used to collect ALL validation // errors that happened during a specific phase, rather than the first one // encountered. -type ValidationPhaseError struct { +type validationPhaseError struct { phase validationPhase errors []error } -func (vpe ValidationPhaseError) Error() string { +func (vpe validationPhaseError) Error() string { msg := fmt.Sprintf("Error during %q phase of README validation:", vpe.phase.String()) for _, e := range vpe.errors { msg += fmt.Sprintf("\n- %v", e) From 18680d0a1570b5165d4cbbf1b873248b7e2513d1 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 15 Apr 2025 16:02:33 +0000 Subject: [PATCH 12/23] chore: add directory validation in separate file --- cmd/readmevalidation/contributors.go | 7 +- cmd/readmevalidation/errors.go | 28 ++++++++ cmd/readmevalidation/main.go | 13 +++- cmd/readmevalidation/readmes.go | 37 +++------- cmd/readmevalidation/repostructure.go | 97 +++++++++++++++++++++++++++ 5 files changed, 147 insertions(+), 35 deletions(-) create mode 100644 cmd/readmevalidation/errors.go create mode 100644 cmd/readmevalidation/repostructure.go diff --git a/cmd/readmevalidation/contributors.go b/cmd/readmevalidation/contributors.go index 8dd6f81..177340c 100644 --- a/cmd/readmevalidation/contributors.go +++ b/cmd/readmevalidation/contributors.go @@ -194,10 +194,6 @@ func validateContributorAvatarURL(avatarURL *string) []error { return problems } -func addFilePathToError(filePath string, err error) error { - return fmt.Errorf("%q: %v", filePath, err) -} - func validateContributorProfile(yml contributorProfile) []error { allProblems := []error{} @@ -313,7 +309,6 @@ func aggregateContributorReadmeFiles() ([]readme, error) { for _, e := range dirEntries { dirPath := path.Join(rootRegistryPath, e.Name()) if !e.IsDir() { - problems = append(problems, fmt.Errorf("detected non-directory file %q at base of main Registry directory", dirPath)) continue } @@ -331,7 +326,7 @@ func aggregateContributorReadmeFiles() ([]readme, error) { if len(problems) != 0 { return nil, validationPhaseError{ - phase: validationPhaseFilesystemRead, + phase: validationPhaseFileLoad, errors: problems, } } diff --git a/cmd/readmevalidation/errors.go b/cmd/readmevalidation/errors.go new file mode 100644 index 0000000..c60e3fc --- /dev/null +++ b/cmd/readmevalidation/errors.go @@ -0,0 +1,28 @@ +package main + +import "fmt" + +var _ error = validationPhaseError{} + +// validationPhaseError represents an error that occurred during a specific +// phase of README validation. It should be used to collect ALL validation +// errors that happened during a specific phase, rather than the first one +// encountered. +type validationPhaseError struct { + phase validationPhase + errors []error +} + +func (vpe validationPhaseError) Error() string { + msg := fmt.Sprintf("Error during %q phase of README validation:", vpe.phase.String()) + for _, e := range vpe.errors { + msg += fmt.Sprintf("\n- %v", e) + } + msg += "\n" + + return msg +} + +func addFilePathToError(filePath string, err error) error { + return fmt.Errorf("%q: %v", filePath, err) +} diff --git a/cmd/readmevalidation/main.go b/cmd/readmevalidation/main.go index f14ed12..d0d7ac3 100644 --- a/cmd/readmevalidation/main.go +++ b/cmd/readmevalidation/main.go @@ -55,10 +55,17 @@ func main() { } else { log.Println("Provided API token does not belong to a Coder employee. Some README validation steps will be skipped compared to when they run in CI.") } - fmt.Printf("actor %q is %s\n", actorUsername, actorOrgStatus.String()) + fmt.Printf("Script GitHub actor %q has Coder organization status %q\n", actorUsername, actorOrgStatus.String()) log.Println("Starting README validation") + // Validate file structure of main README directory + err = validateRepoStructure() + if err != nil { + log.Panic(err) + } + + // Validate contributor README files allReadmeFiles, err := aggregateContributorReadmeFiles() if err != nil { log.Panic(err) @@ -75,4 +82,8 @@ func main() { } log.Println("All relative URLs for READMEs are valid") log.Printf("Processed all READMEs in the %q directory\n", rootRegistryPath) + + // Validate modules + + // Validate templates } diff --git a/cmd/readmevalidation/readmes.go b/cmd/readmevalidation/readmes.go index 47bb196..a8d2bd6 100644 --- a/cmd/readmevalidation/readmes.go +++ b/cmd/readmevalidation/readmes.go @@ -3,7 +3,6 @@ package main import ( "bufio" "errors" - "fmt" "strings" ) @@ -72,12 +71,15 @@ func separateFrontmatter(readmeText string) (string, string, error) { type validationPhase int const ( - // validationPhaseFilesystemRead indicates when a README file is being read - // from the file system - validationPhaseFilesystemRead validationPhase = iota + // + validationPhaseStructureValidation validationPhase = iota - // validationPhaseReadmeParsing indicates when a README's frontmatter is being - // parsed as YAML. This phase does not include YAML validation. + // validationPhaseFileLoad indicates when a README file is being read from + // the file system + validationPhaseFileLoad + + // validationPhaseReadmeParsing indicates when a README's frontmatter is + // being parsed as YAML. This phase does not include YAML validation. validationPhaseReadmeParsing // validationPhaseReadmeValidation indicates when a README's frontmatter is @@ -92,7 +94,7 @@ const ( func (p validationPhase) String() string { switch p { - case validationPhaseFilesystemRead: + case validationPhaseFileLoad: return "Filesystem reading" case validationPhaseReadmeParsing: return "README parsing" @@ -104,24 +106,3 @@ func (p validationPhase) String() string { return "Unknown validation phase" } } - -var _ error = validationPhaseError{} - -// validationPhaseError represents an error that occurred during a specific -// phase of README validation. It should be used to collect ALL validation -// errors that happened during a specific phase, rather than the first one -// encountered. -type validationPhaseError struct { - phase validationPhase - errors []error -} - -func (vpe validationPhaseError) Error() string { - msg := fmt.Sprintf("Error during %q phase of README validation:", vpe.phase.String()) - for _, e := range vpe.errors { - msg += fmt.Sprintf("\n- %v", e) - } - msg += "\n" - - return msg -} diff --git a/cmd/readmevalidation/repostructure.go b/cmd/readmevalidation/repostructure.go new file mode 100644 index 0000000..ccc25c5 --- /dev/null +++ b/cmd/readmevalidation/repostructure.go @@ -0,0 +1,97 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path" +) + +func validateCoderResourceDirectory(directoryPath string) []error { + errs := []error{} + + dir, err := os.Stat(directoryPath) + if err != nil { + // It's valid for a specific resource directory not to exist. It's just + // that if it does exist, it must follow specific rules + if !errors.Is(err, os.ErrNotExist) { + errs = append(errs, addFilePathToError(directoryPath, err)) + } + return errs + } + + if !dir.IsDir() { + errs = append(errs, fmt.Errorf("%q: path is not a directory", directoryPath)) + return errs + } + + files, err := os.ReadDir(directoryPath) + if err != nil { + errs = append(errs, fmt.Errorf("%q: %v", directoryPath, err)) + return errs + } + for _, f := range files { + if !f.IsDir() { + continue + } + + resourceReadmePath := path.Join(directoryPath, f.Name(), "README.md") + _, err := os.Stat(resourceReadmePath) + if err == nil { + continue + } + + if errors.Is(err, os.ErrNotExist) { + errs = append(errs, fmt.Errorf("%q: README file does not exist", resourceReadmePath)) + } else { + errs = append(errs, addFilePathToError(resourceReadmePath, err)) + } + } + + return errs +} + +func validateRegistryDirectory() []error { + dirEntries, err := os.ReadDir(rootRegistryPath) + if err != nil { + return []error{err} + } + + problems := []error{} + for _, e := range dirEntries { + dirPath := path.Join(rootRegistryPath, e.Name()) + if !e.IsDir() { + problems = append(problems, fmt.Errorf("detected non-directory file %q at base of main Registry directory", dirPath)) + continue + } + + readmePath := path.Join(dirPath, "README.md") + _, err := os.Stat(readmePath) + if err != nil { + problems = append(problems, err) + } + + modulesPath := path.Join(dirPath, "modules") + if errs := validateCoderResourceDirectory(modulesPath); len(errs) != 0 { + problems = append(problems, errs...) + } + templatesPath := path.Join(dirPath, "templates") + if errs := validateCoderResourceDirectory(templatesPath); len(errs) != 0 { + problems = append(problems, errs...) + } + } + + return problems +} + +func validateRepoStructure() error { + errs := validateRegistryDirectory() + if len(errs) != 0 { + return validationPhaseError{ + phase: validationPhaseFileLoad, + errors: errs, + } + } + + return nil +} From 17c9667db68079faa10e71f19b3be586d5e81ac1 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 15 Apr 2025 16:35:33 +0000 Subject: [PATCH 13/23] wip: commit progress for module validation --- cmd/readmevalidation/coderResources.go | 100 ++++++++++++++++++ cmd/readmevalidation/readmes.go | 8 +- .../{repostructure.go => repoStructure.go} | 0 3 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 cmd/readmevalidation/coderResources.go rename cmd/readmevalidation/{repostructure.go => repoStructure.go} (100%) diff --git a/cmd/readmevalidation/coderResources.go b/cmd/readmevalidation/coderResources.go new file mode 100644 index 0000000..b1c3928 --- /dev/null +++ b/cmd/readmevalidation/coderResources.go @@ -0,0 +1,100 @@ +package main + +import ( + "errors" + "fmt" + "net/url" + "strings" +) + +type coderResourceFrontmatter struct { + Description string `yaml:"description"` + IconURL string `yaml:"icon"` + DisplayName *string `yaml:"display_name"` + Tags []string `yaml:"tags"` + Verified *bool `yaml:"verified"` +} + +type coderResourceReadme struct { + oldFrontmatter *coderResourceFrontmatter + newFrontmatter *coderResourceFrontmatter + newBody string + moduleName string + filePath string +} + +func validateCoderResourceDisplayName(displayName *string) error { + if displayName == nil { + return nil + } + + if *displayName == "" { + return errors.New("if defined, display_name must not be empty string") + } + + return nil +} + +func validateCoderResourceDescription(description string) error { + if description == "" { + return errors.New("frontmatter description cannot be empty") + } + return nil +} + +func validateCoderResourceIconURL(iconURL string) []error { + problems := []error{} + + if iconURL == "" { + problems = append(problems, errors.New("icon URL cannot be empty")) + return problems + } + + isAbsoluteURL := !strings.HasPrefix(iconURL, ".") && !strings.HasPrefix(iconURL, "/") + if isAbsoluteURL { + if _, err := url.ParseRequestURI(iconURL); err != nil { + problems = append(problems, errors.New("absolute icon URL is not correctly formatted")) + } + if strings.Contains(iconURL, "?") { + problems = append(problems, errors.New("icon URLs cannot contain query parameters")) + } + return problems + } + + // Would normally be skittish about having relative paths like this, but it + // should be safe because we have guarantees about the structure of the + // repo, and where this logic will run + isPermittedRelativeURL := strings.HasPrefix(iconURL, "./") || + strings.HasPrefix(iconURL, "/") || + strings.HasPrefix(iconURL, "../../../.logos") + if !isPermittedRelativeURL { + problems = append(problems, errors.New("relative icon URL must either be scoped to that module's directory, or the top-level /.logos directory")) + } + + return problems +} + +func validateCoderResourceTags(tags []string) error { + if len(tags) == 0 { + return nil + } + + // All of these tags are used for the module/template filter controls in the + // Registry site. Need to make sure they can all be placed in the browser + // URL without issue + invalidTags := []string{} + for _, t := range tags { + if t != url.QueryEscape(t) { + invalidTags = append(invalidTags, t) + } + } + if len(invalidTags) != 0 { + return fmt.Errorf("found invalid tags (tags that cannot be used for filter state in the Registry website): [%s]", strings.Join(invalidTags, ", ")) + } + + return nil +} + +func validateCoderResourceReadmeBody(body string) []error { + return nil +} diff --git a/cmd/readmevalidation/readmes.go b/cmd/readmevalidation/readmes.go index a8d2bd6..00bcca1 100644 --- a/cmd/readmevalidation/readmes.go +++ b/cmd/readmevalidation/readmes.go @@ -71,10 +71,12 @@ func separateFrontmatter(readmeText string) (string, string, error) { type validationPhase int const ( - // - validationPhaseStructureValidation validationPhase = iota + // validationPhaseFileStructureValidation indicates when the entire Registry + // directory is being verified for having all files be placed in the file + // system as expected. + validationPhaseFileStructureValidation validationPhase = iota - // validationPhaseFileLoad indicates when a README file is being read from + // validationPhaseFileLoad indicates when README files are being read from // the file system validationPhaseFileLoad diff --git a/cmd/readmevalidation/repostructure.go b/cmd/readmevalidation/repoStructure.go similarity index 100% rename from cmd/readmevalidation/repostructure.go rename to cmd/readmevalidation/repoStructure.go From aa0b8710d39673926f0b001fa0bd4b65197bdbd9 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 15 Apr 2025 17:29:51 +0000 Subject: [PATCH 14/23] wip: commit progress on validation --- cmd/readmevalidation/coderResources.go | 50 ++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/cmd/readmevalidation/coderResources.go b/cmd/readmevalidation/coderResources.go index b1c3928..374edbb 100644 --- a/cmd/readmevalidation/coderResources.go +++ b/cmd/readmevalidation/coderResources.go @@ -5,22 +5,29 @@ import ( "fmt" "net/url" "strings" + + "coder.com/coder-registry/cmd/github" ) type coderResourceFrontmatter struct { Description string `yaml:"description"` IconURL string `yaml:"icon"` DisplayName *string `yaml:"display_name"` - Tags []string `yaml:"tags"` Verified *bool `yaml:"verified"` + Tags []string `yaml:"tags"` } -type coderResourceReadme struct { +// coderResource represents a generic concept for a Terraform resource used to +// help create Coder workspaces. As of 2025-04-15, this encapsulates both +// Coder Modules and Coder Templates. +type coderResource struct { + name string + filePath string + readmeBody string oldFrontmatter *coderResourceFrontmatter newFrontmatter *coderResourceFrontmatter - newBody string - moduleName string - filePath string + oldIsVerified bool + newIsVerified bool } func validateCoderResourceDisplayName(displayName *string) error { @@ -95,6 +102,37 @@ func validateCoderResourceTags(tags []string) error { return nil } -func validateCoderResourceReadmeBody(body string) []error { +func validateCoderResourceVerifiedStatus(oldVerified bool, newVerified bool, actorOrgStatus github.OrgStatus) error { + // If the actor making the changes is an employee of Coder, any changes are + // assumed to be valid + if actorOrgStatus == github.OrgStatusMember { + return nil + } + + // Right now, because we collapse the omitted/nil case and false together, + // the only field transition that's allowed is if the verified statuses are + // exactly the same (which includes the field going from omitted to + // explicitly false, or vice-versa). + isPermittedChangeForNonEmployee := oldVerified == newVerified + if isPermittedChangeForNonEmployee { + return nil + } + + return fmt.Errorf("actor with status %q is not allowed to flip verified status from %t to %t", actorOrgStatus.String(), oldVerified, newVerified) +} + +// Todo: once we decide on how we want the README frontmatter to be formatted +// for the Embedded Registry work, update this function to validate that the +// correct Terraform code snippets are included in the README and are actually +// valid Terraform +func validateCoderResourceReadmeBody(body string) error { + trimmed := strings.TrimSpace(body) + if !strings.HasPrefix(trimmed, "# ") { + return errors.New("README body must start with ATX-style h1 header (i.e., \"# \")") + } + return nil +} + +func validateCoderResource(resource coderResource) []error { return nil } From e8885060639bae65f21622166a09f1ac4e25f261 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 15 Apr 2025 18:10:32 +0000 Subject: [PATCH 15/23] wip: get concurrency stubs set up --- cmd/readmevalidation/coderResources.go | 49 +++++++++++++++++-- cmd/readmevalidation/contributors.go | 4 +- cmd/readmevalidation/main.go | 68 ++++++++++++++++++++------ 3 files changed, 98 insertions(+), 23 deletions(-) diff --git a/cmd/readmevalidation/coderResources.go b/cmd/readmevalidation/coderResources.go index 374edbb..76e4409 100644 --- a/cmd/readmevalidation/coderResources.go +++ b/cmd/readmevalidation/coderResources.go @@ -19,11 +19,12 @@ type coderResourceFrontmatter struct { // coderResource represents a generic concept for a Terraform resource used to // help create Coder workspaces. As of 2025-04-15, this encapsulates both -// Coder Modules and Coder Templates. +// Coder Modules and Coder Templates. If the newReadmeBody and newFrontmatter +// fields are nil, that represents that the file has been deleted type coderResource struct { name string filePath string - readmeBody string + newReadmeBody *string oldFrontmatter *coderResourceFrontmatter newFrontmatter *coderResourceFrontmatter oldIsVerified bool @@ -124,7 +125,8 @@ func validateCoderResourceVerifiedStatus(oldVerified bool, newVerified bool, act // Todo: once we decide on how we want the README frontmatter to be formatted // for the Embedded Registry work, update this function to validate that the // correct Terraform code snippets are included in the README and are actually -// valid Terraform +// valid Terraform. Might also want to validate that each header follows proper +// hierarchy (i.e., not jumping from h1 to h3 because you think it looks nicer) func validateCoderResourceReadmeBody(body string) error { trimmed := strings.TrimSpace(body) if !strings.HasPrefix(trimmed, "# ") { @@ -133,6 +135,45 @@ func validateCoderResourceReadmeBody(body string) error { return nil } -func validateCoderResource(resource coderResource) []error { +func validateCoderResourceChanges(resource coderResource, actorOrgStatus github.OrgStatus) []error { + var problems []error + + if resource.newReadmeBody != nil { + if err := validateCoderResourceReadmeBody(*resource.newReadmeBody); err != nil { + problems = append(problems, addFilePathToError(resource.filePath, err)) + } + } + + if resource.newFrontmatter != nil { + if err := validateCoderResourceDisplayName(resource.newFrontmatter.DisplayName); err != nil { + problems = append(problems, addFilePathToError(resource.filePath, err)) + } + if err := validateCoderResourceDescription(resource.newFrontmatter.Description); err != nil { + problems = append(problems, addFilePathToError(resource.filePath, err)) + } + if err := validateCoderResourceTags(resource.newFrontmatter.Tags); err != nil { + problems = append(problems, addFilePathToError(resource.filePath, err)) + } + if err := validateCoderResourceVerifiedStatus(resource.oldIsVerified, resource.newIsVerified, actorOrgStatus); err != nil { + problems = append(problems, addFilePathToError(resource.filePath, err)) + } + + for _, err := range validateCoderResourceIconURL(resource.newFrontmatter.IconURL) { + problems = append(problems, addFilePathToError(resource.filePath, err)) + } + } + + return problems +} + +func parseCoderResourceFiles(oldReadmeFiles []readme, newReadmeFiles []readme) (map[string]coderResource, error) { + return nil, nil +} + +func validateCoderResourceRelativeUrls(map[string]coderResource) []error { return nil } + +func aggregateCoderResourceReadmeFiles() ([]readme, error) { + return nil, nil +} diff --git a/cmd/readmevalidation/contributors.go b/cmd/readmevalidation/contributors.go index 177340c..782cc0c 100644 --- a/cmd/readmevalidation/contributors.go +++ b/cmd/readmevalidation/contributors.go @@ -334,9 +334,7 @@ func aggregateContributorReadmeFiles() ([]readme, error) { return allReadmeFiles, nil } -func validateContributorRelativeUrls( - contributors map[string]contributorProfile, -) error { +func validateContributorRelativeUrls(contributors map[string]contributorProfile) error { // This function only validates relative avatar URLs for now, but it can be // beefed up to validate more in the future problems := []error{} diff --git a/cmd/readmevalidation/main.go b/cmd/readmevalidation/main.go index d0d7ac3..28fdb50 100644 --- a/cmd/readmevalidation/main.go +++ b/cmd/readmevalidation/main.go @@ -9,6 +9,8 @@ package main import ( "fmt" "log" + "os" + "sync" "coder.com/coder-registry/cmd/github" "github.com/joho/godotenv" @@ -60,30 +62,64 @@ func main() { log.Println("Starting README validation") // Validate file structure of main README directory + log.Println("Validating directory structure of the README directory") err = validateRepoStructure() if err != nil { log.Panic(err) } + // Set up concurrency for validating each category of README file + var readmeValidationErrors []error + errChan := make(chan error, 1) + wg := sync.WaitGroup{} + go func() { + for err := range errChan { + readmeValidationErrors = append(readmeValidationErrors, err) + } + }() + // Validate contributor README files - allReadmeFiles, err := aggregateContributorReadmeFiles() - if err != nil { - log.Panic(err) - } - log.Printf("Processing %d README files\n", len(allReadmeFiles)) - contributors, err := parseContributorFiles(allReadmeFiles) - log.Printf("Processed %d README files as valid contributor profiles", len(contributors)) - if err != nil { - log.Panic(err) - } - err = validateContributorRelativeUrls(contributors) - if err != nil { - log.Panic(err) - } - log.Println("All relative URLs for READMEs are valid") - log.Printf("Processed all READMEs in the %q directory\n", rootRegistryPath) + wg.Add(1) + go func() { + defer wg.Done() + + allReadmeFiles, err := aggregateContributorReadmeFiles() + if err != nil { + log.Panic(err) + } + log.Printf("Processing %d README files\n", len(allReadmeFiles)) + contributors, err := parseContributorFiles(allReadmeFiles) + log.Printf("Processed %d README files as valid contributor profiles", len(contributors)) + if err != nil { + log.Panic(err) + } + err = validateContributorRelativeUrls(contributors) + if err != nil { + log.Panic(err) + } + log.Println("All relative URLs for READMEs are valid") + log.Printf("Processed all READMEs in the %q directory\n", rootRegistryPath) + }() // Validate modules + wg.Add(1) + go func() { + defer wg.Done() + }() // Validate templates + wg.Add(1) + go func() { + defer wg.Done() + }() + + // Clean up and log errors + wg.Wait() + close(errChan) + for _, err := range readmeValidationErrors { + log.Println(err) + } + if len(readmeValidationErrors) != 0 { + os.Exit(1) + } } From 19226af0674e855dbc4046ac543a18e1d3c732f0 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 15 Apr 2025 23:56:48 +0000 Subject: [PATCH 16/23] wip: commit more progress --- .env.example | 4 -- cmd/github/github.go | 54 ++++++--------------- cmd/readmevalidation/coderResources.go | 65 +++++++++++++++++++++++-- cmd/readmevalidation/github.go | 33 +++++++++++++ cmd/readmevalidation/main.go | 48 +++++++++++++++--- cmd/readmevalidation/repoStructure.go | 46 ++++++++++-------- go.mod | 24 ++++++++- go.sum | 67 ++++++++++++++++++++++++++ 8 files changed, 266 insertions(+), 75 deletions(-) create mode 100644 cmd/readmevalidation/github.go diff --git a/.env.example b/.env.example index 6bf034b..c8c8414 100644 --- a/.env.example +++ b/.env.example @@ -4,10 +4,6 @@ # can set this to your GitHub username. CI_ACTOR= -# This is the Git ref that you want to merge into the main branch. In local -# development, this should be set to the value of the branch you're working from -CI_BASE_REF= - # This is the configurable base URL for accessing the GitHub REST API. This # value will be injected by the CI script's Actions context, but if the value is # not defined (either in CI or when running locally), "https://api.github.com/" diff --git a/cmd/github/github.go b/cmd/github/github.go index 00b3f3a..acc7184 100644 --- a/cmd/github/github.go +++ b/cmd/github/github.go @@ -7,46 +7,12 @@ import ( "errors" "fmt" "io" - "log" "net/http" - "os" "time" ) const defaultGithubAPIBaseRoute = "https://api.github.com/" -const ( - actionsActorKey = "CI_ACTOR" - actionsBaseRefKey = "CI_BASE_REF" -) - -const ( - githubAPIURLKey = "GITHUB_API_URL" - githubAPITokenKey = "GITHUB_API_TOKEN" -) - -// ActionsActor returns the username of the GitHub user who triggered the -// current CI run as part of GitHub Actions. The value must be loaded into the -// env as part of the Github Actions YAML file, or else the function fails. -func ActionsActor() (string, error) { - username := os.Getenv(actionsActorKey) - if username == "" { - return "", fmt.Errorf("value for %q is not in env. If running from CI, please add value via ci.yaml file", actionsActorKey) - } - return username, nil -} - -// BaseRef returns the name of the base ref for the Git branch that will be -// merged into the main branch. -func BaseRef() (string, error) { - baseRef := os.Getenv(actionsBaseRefKey) - if baseRef == "" { - return "", fmt.Errorf("value for %q is not in env. If running from CI, please add value via ci.yaml file", actionsBaseRefKey) - } - - return baseRef, nil -} - // Client is a reusable REST client for making requests to the GitHub API. // It should be instantiated via NewGithubClient type Client struct { @@ -55,19 +21,25 @@ type Client struct { httpClient http.Client } -// NewClient instantiates a GitHub client -func NewClient() (*Client, error) { +// ClientInit is used to instantiate a new client. If the value of BaseURL is +// not defined, a default value of "https://api.github.com/" is used instead +type ClientInit struct { + BaseURL string + APIToken string +} + +// NewClient instantiates a GitHub client. If the baseURL is +func NewClient(init ClientInit) (*Client, error) { // Considered letting the user continue on with no token and more aggressive // rate-limiting, but from experimentation, the non-authenticated experience // hit the rate limits really quickly, and had a lot of restrictions - apiToken := os.Getenv(githubAPITokenKey) + apiToken := init.APIToken if apiToken == "" { - return nil, fmt.Errorf("missing env variable %q", githubAPITokenKey) + return nil, errors.New("API token is missing") } - baseURL := os.Getenv(githubAPIURLKey) + baseURL := init.BaseURL if baseURL == "" { - log.Printf("env variable %q is not defined. Falling back to %q\n", githubAPIURLKey, defaultGithubAPIBaseRoute) baseURL = defaultGithubAPIBaseRoute } @@ -129,9 +101,11 @@ const ( // could not be determined. It is the zero value of the OrgStatus type, and // any users with this value should be treated as completely untrusted OrgStatusIndeterminate = iota + // OrgStatusNonMember indicates when a user is definitely NOT part of an // organization OrgStatusNonMember + // OrgStatusMember indicates when a user is a member of a Github // organization OrgStatusMember diff --git a/cmd/readmevalidation/coderResources.go b/cmd/readmevalidation/coderResources.go index 76e4409..d4a6c39 100644 --- a/cmd/readmevalidation/coderResources.go +++ b/cmd/readmevalidation/coderResources.go @@ -4,11 +4,16 @@ import ( "errors" "fmt" "net/url" + "os" + "path" + "slices" "strings" "coder.com/coder-registry/cmd/github" ) +var supportedResourceTypes = []string{"modules", "templates"} + type coderResourceFrontmatter struct { Description string `yaml:"description"` IconURL string `yaml:"icon"` @@ -166,7 +171,7 @@ func validateCoderResourceChanges(resource coderResource, actorOrgStatus github. return problems } -func parseCoderResourceFiles(oldReadmeFiles []readme, newReadmeFiles []readme) (map[string]coderResource, error) { +func parseCoderResourceFiles(oldReadmeFiles []readme, newReadmeFiles []readme, actorOrgStatus github.OrgStatus) (map[string]coderResource, error) { return nil, nil } @@ -174,6 +179,60 @@ func validateCoderResourceRelativeUrls(map[string]coderResource) []error { return nil } -func aggregateCoderResourceReadmeFiles() ([]readme, error) { - return nil, nil +func aggregateCoderResourceReadmeFiles(resourceDirectoryName string) ([]readme, error) { + if !slices.Contains(supportedResourceTypes, resourceDirectoryName) { + return nil, fmt.Errorf("%q is not a supported resource type. Must be one of [%s]", resourceDirectoryName, strings.Join(supportedResourceTypes, ", ")) + } + + registryFiles, err := os.ReadDir(rootRegistryPath) + if err != nil { + return nil, err + } + + var allReadmeFiles []readme + var problems []error + for _, f := range registryFiles { + if !f.IsDir() { + continue + } + + resourceDirPath := path.Join(rootRegistryPath, f.Name(), resourceDirectoryName) + resourceFiles, err := os.ReadDir(resourceDirPath) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + problems = append(problems, err) + } + continue + } + + for _, resFile := range resourceFiles { + // Not sure if we want to allow non-directories to live inside of + // main directories like /modules or /templates, but we can tighten + // things up later + if !resFile.IsDir() { + continue + } + + readmePath := path.Join(resourceDirPath, resFile.Name(), "README.md") + rawRm, err := os.ReadFile(readmePath) + if err != nil { + problems = append(problems, err) + continue + } + allReadmeFiles = append(allReadmeFiles, readme{ + filePath: readmePath, + rawText: string(rawRm), + }) + + } + } + + if len(problems) != 0 { + return nil, validationPhaseError{ + phase: validationPhaseFileLoad, + errors: problems, + } + } + + return allReadmeFiles, nil } diff --git a/cmd/readmevalidation/github.go b/cmd/readmevalidation/github.go new file mode 100644 index 0000000..efb08f7 --- /dev/null +++ b/cmd/readmevalidation/github.go @@ -0,0 +1,33 @@ +package main + +import ( + "fmt" + "os" +) + +const actionsActorKey = "CI_ACTOR" + +const ( + githubAPIBaseURLKey = "GITHUB_API_URL" + githubAPITokenKey = "GITHUB_API_TOKEN" +) + +// actionsActor returns the username of the GitHub user who triggered the +// current CI run as part of GitHub Actions. It is expected that this value be +// set using a local .env file in local development, and set via GitHub Actions +// context during CI. +func actionsActor() (string, error) { + username := os.Getenv(actionsActorKey) + if username == "" { + return "", fmt.Errorf("value for %q is not in env. If running from CI, please add value via ci.yaml file", actionsActorKey) + } + return username, nil +} + +func githubAPIToken() (string, error) { + token := os.Getenv(githubAPITokenKey) + if token == "" { + return "", fmt.Errorf("value for %q is not in env. If running from CI, please add value via ci.yaml file", githubAPITokenKey) + } + return token, nil +} diff --git a/cmd/readmevalidation/main.go b/cmd/readmevalidation/main.go index 28fdb50..c4fa13e 100644 --- a/cmd/readmevalidation/main.go +++ b/cmd/readmevalidation/main.go @@ -13,30 +13,34 @@ import ( "sync" "coder.com/coder-registry/cmd/github" + "github.com/go-git/go-git/v5" "github.com/joho/godotenv" ) func main() { - // Do basic setup log.Println("Beginning README file validation") + + // Do basic setup err := godotenv.Load() if err != nil { log.Panic(err) } - actorUsername, err := github.ActionsActor() + actorUsername, err := actionsActor() if err != nil { log.Panic(err) } - baseRef, err := github.BaseRef() + ghAPIToken, err := githubAPIToken() if err != nil { log.Panic(err) } - log.Printf("Using branch %q for validation comparison", baseRef) // Retrieve data necessary from the GitHub API to help determine whether // certain field changes are allowed log.Printf("Using GitHub API to determine what fields can be set by user %q\n", actorUsername) - client, err := github.NewClient() + client, err := github.NewClient(github.ClientInit{ + BaseURL: os.Getenv(githubAPIBaseURLKey), + APIToken: ghAPIToken, + }) if err != nil { log.Panic(err) } @@ -59,6 +63,7 @@ func main() { } fmt.Printf("Script GitHub actor %q has Coder organization status %q\n", actorUsername, actorOrgStatus.String()) + // Start main validation log.Println("Starting README validation") // Validate file structure of main README directory @@ -85,17 +90,20 @@ func main() { allReadmeFiles, err := aggregateContributorReadmeFiles() if err != nil { - log.Panic(err) + errChan <- err + return } log.Printf("Processing %d README files\n", len(allReadmeFiles)) contributors, err := parseContributorFiles(allReadmeFiles) log.Printf("Processed %d README files as valid contributor profiles", len(contributors)) if err != nil { - log.Panic(err) + errChan <- err + return } err = validateContributorRelativeUrls(contributors) if err != nil { - log.Panic(err) + errChan <- err + return } log.Println("All relative URLs for READMEs are valid") log.Printf("Processed all READMEs in the %q directory\n", rootRegistryPath) @@ -105,6 +113,30 @@ func main() { wg.Add(1) go func() { defer wg.Done() + + baseRefReadmeFiles, err := aggregateCoderResourceReadmeFiles("modules") + if err != nil { + errChan <- err + return + } + fmt.Printf("------ got %d back\n", len(baseRefReadmeFiles)) + + repo, err := git.PlainOpenWithOptions(".", &git.PlainOpenOptions{ + DetectDotGit: false, + EnableDotGitCommonDir: false, + }) + if err != nil { + errChan <- err + return + } + + head, err := repo.Head() + if err != nil { + errChan <- err + return + } + activeBranchName := head.Name().Short() + fmt.Println("-----", activeBranchName) }() // Validate templates diff --git a/cmd/readmevalidation/repoStructure.go b/cmd/readmevalidation/repoStructure.go index ccc25c5..328f5a0 100644 --- a/cmd/readmevalidation/repoStructure.go +++ b/cmd/readmevalidation/repoStructure.go @@ -7,27 +7,27 @@ import ( "path" ) -func validateCoderResourceDirectory(directoryPath string) []error { +func validateCoderResourceSubdirectory(dirPath string) []error { errs := []error{} - dir, err := os.Stat(directoryPath) + dir, err := os.Stat(dirPath) if err != nil { // It's valid for a specific resource directory not to exist. It's just // that if it does exist, it must follow specific rules if !errors.Is(err, os.ErrNotExist) { - errs = append(errs, addFilePathToError(directoryPath, err)) + errs = append(errs, addFilePathToError(dirPath, err)) } return errs } if !dir.IsDir() { - errs = append(errs, fmt.Errorf("%q: path is not a directory", directoryPath)) + errs = append(errs, fmt.Errorf("%q: path is not a directory", dirPath)) return errs } - files, err := os.ReadDir(directoryPath) + files, err := os.ReadDir(dirPath) if err != nil { - errs = append(errs, fmt.Errorf("%q: %v", directoryPath, err)) + errs = append(errs, fmt.Errorf("%q: %v", dirPath, err)) return errs } for _, f := range files { @@ -35,7 +35,7 @@ func validateCoderResourceDirectory(directoryPath string) []error { continue } - resourceReadmePath := path.Join(directoryPath, f.Name(), "README.md") + resourceReadmePath := path.Join(dirPath, f.Name(), "README.md") _, err := os.Stat(resourceReadmePath) if err == nil { continue @@ -71,13 +71,11 @@ func validateRegistryDirectory() []error { problems = append(problems, err) } - modulesPath := path.Join(dirPath, "modules") - if errs := validateCoderResourceDirectory(modulesPath); len(errs) != 0 { - problems = append(problems, errs...) - } - templatesPath := path.Join(dirPath, "templates") - if errs := validateCoderResourceDirectory(templatesPath); len(errs) != 0 { - problems = append(problems, errs...) + for _, rType := range supportedResourceTypes { + resourcePath := path.Join(dirPath, rType) + if errs := validateCoderResourceSubdirectory(resourcePath); len(errs) != 0 { + problems = append(problems, errs...) + } } } @@ -85,13 +83,23 @@ func validateRegistryDirectory() []error { } func validateRepoStructure() error { - errs := validateRegistryDirectory() - if len(errs) != 0 { + var problems []error + if errs := validateRegistryDirectory(); len(errs) != 0 { + problems = append(problems, errs...) + } + + _, err := os.Stat("./.logos") + if err != nil { + problems = append(problems, err) + } + + // Todo: figure out what other directories and decide what other invariants + // we want to set for them + if len(problems) != 0 { return validationPhaseError{ - phase: validationPhaseFileLoad, - errors: errs, + phase: validationPhaseFileStructureValidation, + errors: problems, } } - return nil } diff --git a/go.mod b/go.mod index 61231b9..0b88cd4 100644 --- a/go.mod +++ b/go.mod @@ -4,4 +4,26 @@ go 1.23.2 require gopkg.in/yaml.v3 v3.0.1 -require github.com/joho/godotenv v1.5.1 // indirect +require ( + dario.cat/mergo v1.0.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect + github.com/cloudflare/circl v1.6.1 // indirect + github.com/cyphar/filepath-securejoin v0.4.1 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.6.2 // indirect + github.com/go-git/go-git/v5 v5.16.0 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/pjbgf/sha1cd v0.3.2 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/sys v0.32.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect +) diff --git a/go.sum b/go.sum index e536c30..cbdb91f 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,73 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= +github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= +github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= +github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= +github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM= +github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU= +github.com/go-git/go-git/v5 v5.16.0 h1:k3kuOEpkc0DeY7xlL6NaaNg39xdgQbtH5mwCafHO9AQ= +github.com/go-git/go-git/v5 v5.16.0/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= +github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From f23bbca2e77c3862e5a9f857a84faa29721e9deb Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Wed, 16 Apr 2025 00:03:01 +0000 Subject: [PATCH 17/23] refactor: start making main() leaner --- cmd/readmevalidation/contributors.go | 23 +++++++++++++++++++++++ cmd/readmevalidation/main.go | 21 +-------------------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/cmd/readmevalidation/contributors.go b/cmd/readmevalidation/contributors.go index 782cc0c..f8447c5 100644 --- a/cmd/readmevalidation/contributors.go +++ b/cmd/readmevalidation/contributors.go @@ -3,6 +3,7 @@ package main import ( "errors" "fmt" + "log" "net/url" "os" "path" @@ -371,3 +372,25 @@ func validateContributorRelativeUrls(contributors map[string]contributorProfile) errors: problems, } } + +func validateAllContributors(errChan chan<- error) { + allReadmeFiles, err := aggregateContributorReadmeFiles() + if err != nil { + errChan <- err + return + } + log.Printf("Processing %d README files\n", len(allReadmeFiles)) + contributors, err := parseContributorFiles(allReadmeFiles) + log.Printf("Processed %d README files as valid contributor profiles", len(contributors)) + if err != nil { + errChan <- err + return + } + err = validateContributorRelativeUrls(contributors) + if err != nil { + errChan <- err + return + } + log.Println("All relative URLs for READMEs are valid") + log.Printf("Processed all READMEs in the %q directory\n", rootRegistryPath) +} diff --git a/cmd/readmevalidation/main.go b/cmd/readmevalidation/main.go index c4fa13e..1471a65 100644 --- a/cmd/readmevalidation/main.go +++ b/cmd/readmevalidation/main.go @@ -87,26 +87,7 @@ func main() { wg.Add(1) go func() { defer wg.Done() - - allReadmeFiles, err := aggregateContributorReadmeFiles() - if err != nil { - errChan <- err - return - } - log.Printf("Processing %d README files\n", len(allReadmeFiles)) - contributors, err := parseContributorFiles(allReadmeFiles) - log.Printf("Processed %d README files as valid contributor profiles", len(contributors)) - if err != nil { - errChan <- err - return - } - err = validateContributorRelativeUrls(contributors) - if err != nil { - errChan <- err - return - } - log.Println("All relative URLs for READMEs are valid") - log.Printf("Processed all READMEs in the %q directory\n", rootRegistryPath) + validateAllContributors(errChan) }() // Validate modules From 135d5c811155a538447790c0392751c40db27b6d Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Wed, 16 Apr 2025 01:44:21 +0000 Subject: [PATCH 18/23] fix: remove potential race condition --- cmd/readmevalidation/contributors.go | 13 ++++++------- cmd/readmevalidation/main.go | 21 +++++++++++++++++---- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/cmd/readmevalidation/contributors.go b/cmd/readmevalidation/contributors.go index f8447c5..aa3676d 100644 --- a/cmd/readmevalidation/contributors.go +++ b/cmd/readmevalidation/contributors.go @@ -373,24 +373,23 @@ func validateContributorRelativeUrls(contributors map[string]contributorProfile) } } -func validateAllContributors(errChan chan<- error) { +func validateAllContributors() error { allReadmeFiles, err := aggregateContributorReadmeFiles() if err != nil { - errChan <- err - return + return err } log.Printf("Processing %d README files\n", len(allReadmeFiles)) contributors, err := parseContributorFiles(allReadmeFiles) log.Printf("Processed %d README files as valid contributor profiles", len(contributors)) if err != nil { - errChan <- err - return + return err } err = validateContributorRelativeUrls(contributors) if err != nil { - errChan <- err - return + return err } log.Println("All relative URLs for READMEs are valid") log.Printf("Processed all READMEs in the %q directory\n", rootRegistryPath) + + return nil } diff --git a/cmd/readmevalidation/main.go b/cmd/readmevalidation/main.go index 1471a65..b4f108d 100644 --- a/cmd/readmevalidation/main.go +++ b/cmd/readmevalidation/main.go @@ -14,6 +14,7 @@ import ( "coder.com/coder-registry/cmd/github" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" "github.com/joho/godotenv" ) @@ -66,7 +67,10 @@ func main() { // Start main validation log.Println("Starting README validation") - // Validate file structure of main README directory + // Validate file structure of main README directory. Have to do this + // synchronously and before everything else, or else there's no way to for + // the other main validation functions can't make any safe assumptions + // about where they should look in the repo log.Println("Validating directory structure of the README directory") err = validateRepoStructure() if err != nil { @@ -76,18 +80,22 @@ func main() { // Set up concurrency for validating each category of README file var readmeValidationErrors []error errChan := make(chan error, 1) + doneChan := make(chan struct{}) wg := sync.WaitGroup{} go func() { for err := range errChan { readmeValidationErrors = append(readmeValidationErrors, err) } + close(doneChan) }() // Validate contributor README files wg.Add(1) go func() { defer wg.Done() - validateAllContributors(errChan) + if err := validateAllContributors(); err != nil { + errChan <- fmt.Errorf("contributor validation: %v", err) + } }() // Validate modules @@ -117,7 +125,11 @@ func main() { return } activeBranchName := head.Name().Short() - fmt.Println("-----", activeBranchName) + _, err = repo.Reference(plumbing.ReferenceName(activeBranchName), true) + if err != nil { + errChan <- err + return + } }() // Validate templates @@ -126,9 +138,10 @@ func main() { defer wg.Done() }() - // Clean up and log errors + // Clean up and then log errors wg.Wait() close(errChan) + <-doneChan for _, err := range readmeValidationErrors { log.Println(err) } From 6eef059e217c236deb1ac8c355b1feca564ac9cf Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Wed, 16 Apr 2025 01:58:36 +0000 Subject: [PATCH 19/23] wip: commit progress on git manipulation --- cmd/readmevalidation/main.go | 26 ++++++++++++++++++++++---- cmd/readmevalidation/repoStructure.go | 4 ++-- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/cmd/readmevalidation/main.go b/cmd/readmevalidation/main.go index b4f108d..b4c393e 100644 --- a/cmd/readmevalidation/main.go +++ b/cmd/readmevalidation/main.go @@ -125,11 +125,25 @@ func main() { return } activeBranchName := head.Name().Short() - _, err = repo.Reference(plumbing.ReferenceName(activeBranchName), true) + fmt.Println("Found ", activeBranchName) + + wt, err := repo.Worktree() + if err != nil { + errChan <- err + return + } + err = wt.Checkout(&git.CheckoutOptions{ + Branch: plumbing.ReferenceName(activeBranchName), + Create: false, + Force: false, + Keep: true, + }) if err != nil { errChan <- err return } + + fmt.Println("Got here!") }() // Validate templates @@ -142,10 +156,14 @@ func main() { wg.Wait() close(errChan) <-doneChan + if len(readmeValidationErrors) == 0 { + log.Println("All validation was successful") + return + } + + fmt.Println("---\nEncountered the following problems") for _, err := range readmeValidationErrors { log.Println(err) } - if len(readmeValidationErrors) != 0 { - os.Exit(1) - } + os.Exit(1) } diff --git a/cmd/readmevalidation/repoStructure.go b/cmd/readmevalidation/repoStructure.go index 328f5a0..732a2a4 100644 --- a/cmd/readmevalidation/repoStructure.go +++ b/cmd/readmevalidation/repoStructure.go @@ -93,8 +93,8 @@ func validateRepoStructure() error { problems = append(problems, err) } - // Todo: figure out what other directories and decide what other invariants - // we want to set for them + // Todo: figure out what other directories we want to make guarantees for + // and add them to this function if len(problems) != 0 { return validationPhaseError{ phase: validationPhaseFileStructureValidation, From a00a9ce589abd9c3f76f50c63c4e983e5f151ea2 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Wed, 16 Apr 2025 02:17:15 +0000 Subject: [PATCH 20/23] wip: commit progress --- cmd/readmevalidation/main.go | 87 ++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 38 deletions(-) diff --git a/cmd/readmevalidation/main.go b/cmd/readmevalidation/main.go index b4c393e..d1c38e3 100644 --- a/cmd/readmevalidation/main.go +++ b/cmd/readmevalidation/main.go @@ -103,47 +103,57 @@ func main() { go func() { defer wg.Done() - baseRefReadmeFiles, err := aggregateCoderResourceReadmeFiles("modules") - if err != nil { - errChan <- err - return - } - fmt.Printf("------ got %d back\n", len(baseRefReadmeFiles)) - - repo, err := git.PlainOpenWithOptions(".", &git.PlainOpenOptions{ - DetectDotGit: false, - EnableDotGitCommonDir: false, - }) - if err != nil { - errChan <- err - return - } - - head, err := repo.Head() - if err != nil { - errChan <- err - return + refactorLater := func() error { + baseRefReadmeFiles, err := aggregateCoderResourceReadmeFiles("modules") + if err != nil { + return err + } + fmt.Printf("------ got %d back\n", len(baseRefReadmeFiles)) + + repo, err := git.PlainOpenWithOptions(".", &git.PlainOpenOptions{ + DetectDotGit: false, + EnableDotGitCommonDir: false, + }) + if err != nil { + return err + } + + head, err := repo.Head() + if err != nil { + return err + } + activeBranchName := head.Name().Short() + fmt.Println("yeah...") + + tree, err := repo.Worktree() + if err != nil { + return err + } + err = tree.Checkout(&git.CheckoutOptions{ + Branch: plumbing.NewBranchReferenceName(activeBranchName), + Create: false, + Force: false, + Keep: true, + }) + if err != nil { + return err + } + + fmt.Println("Got here!") + files, _ := tree.Filesystem.ReadDir(".") + for _, f := range files { + if f.IsDir() { + fmt.Println(f.Name()) + } + } + + return nil } - activeBranchName := head.Name().Short() - fmt.Println("Found ", activeBranchName) - wt, err := repo.Worktree() - if err != nil { - errChan <- err - return - } - err = wt.Checkout(&git.CheckoutOptions{ - Branch: plumbing.ReferenceName(activeBranchName), - Create: false, - Force: false, - Keep: true, - }) - if err != nil { - errChan <- err - return + if err := refactorLater(); err != nil { + errChan <- fmt.Errorf("module validation: %v", err) } - fmt.Println("Got here!") }() // Validate templates @@ -161,7 +171,8 @@ func main() { return } - fmt.Println("---\nEncountered the following problems") + fmt.Println("---") + fmt.Println("Encountered the following problems") for _, err := range readmeValidationErrors { log.Println(err) } From f10f5a44032e9cc37ec2e30b31727fbdb2bc578e Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Wed, 16 Apr 2025 02:44:34 +0000 Subject: [PATCH 21/23] chore: add sample data --- .gitignore | 3 +- cmd/readmevalidation/coderResources.go | 7 + cmd/readmevalidation/main.go | 8 +- cmd/readmevalidation/repoStructure.go | 6 + registry/coder/modules/claude-code/README.md | 113 ++++++++++++ registry/coder/modules/claude-code/main.tf | 170 +++++++++++++++++++ registry/coder/modules/cursor/README.md | 36 ++++ registry/coder/modules/cursor/main.test.ts | 88 ++++++++++ registry/coder/modules/cursor/main.tf | 62 +++++++ 9 files changed, 488 insertions(+), 5 deletions(-) create mode 100644 registry/coder/modules/claude-code/README.md create mode 100644 registry/coder/modules/claude-code/main.tf create mode 100644 registry/coder/modules/cursor/README.md create mode 100644 registry/coder/modules/cursor/main.test.ts create mode 100644 registry/coder/modules/cursor/main.tf diff --git a/.gitignore b/.gitignore index 0f945ce..2922fed 100644 --- a/.gitignore +++ b/.gitignore @@ -135,5 +135,6 @@ dist .yarn/install-state.gz .pnp.* -# Script output +# Things needed for CI /readmevalidation +/readmevalidation-git diff --git a/cmd/readmevalidation/coderResources.go b/cmd/readmevalidation/coderResources.go index d4a6c39..eda5fd6 100644 --- a/cmd/readmevalidation/coderResources.go +++ b/cmd/readmevalidation/coderResources.go @@ -12,6 +12,13 @@ import ( "coder.com/coder-registry/cmd/github" ) +// dummyGitDirectory is the directory that a full version of the Registry will +// be cloned into during CI. The CI needs to use Git history to validate +// certain README files, and using the root branch itself (even though it's +// fully equivalent) has a risk of breaking other CI steps when switching +// branches. Better to make a full isolated copy and manipulate that. +const dummyGitDirectory = "./readmevalidation-git" + var supportedResourceTypes = []string{"modules", "templates"} type coderResourceFrontmatter struct { diff --git a/cmd/readmevalidation/main.go b/cmd/readmevalidation/main.go index d1c38e3..f58fc47 100644 --- a/cmd/readmevalidation/main.go +++ b/cmd/readmevalidation/main.go @@ -15,6 +15,7 @@ import ( "coder.com/coder-registry/cmd/github" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/joho/godotenv" ) @@ -110,9 +111,9 @@ func main() { } fmt.Printf("------ got %d back\n", len(baseRefReadmeFiles)) - repo, err := git.PlainOpenWithOptions(".", &git.PlainOpenOptions{ - DetectDotGit: false, - EnableDotGitCommonDir: false, + repo, err := git.PlainClone(dummyGitDirectory, true, &git.CloneOptions{ + URL: "https://github.com/coder/registry", + Auth: &http.BasicAuth{}, }) if err != nil { return err @@ -123,7 +124,6 @@ func main() { return err } activeBranchName := head.Name().Short() - fmt.Println("yeah...") tree, err := repo.Worktree() if err != nil { diff --git a/cmd/readmevalidation/repoStructure.go b/cmd/readmevalidation/repoStructure.go index 732a2a4..ebc8809 100644 --- a/cmd/readmevalidation/repoStructure.go +++ b/cmd/readmevalidation/repoStructure.go @@ -71,6 +71,12 @@ func validateRegistryDirectory() []error { problems = append(problems, err) } + mainTerraformPath := path.Join(dirPath, "main.tf") + _, err = os.Stat(mainTerraformPath) + if err != nil { + problems = append(problems, err) + } + for _, rType := range supportedResourceTypes { resourcePath := path.Join(dirPath, rType) if errs := validateCoderResourceSubdirectory(resourcePath); len(errs) != 0 { diff --git a/registry/coder/modules/claude-code/README.md b/registry/coder/modules/claude-code/README.md new file mode 100644 index 0000000..2cb506b --- /dev/null +++ b/registry/coder/modules/claude-code/README.md @@ -0,0 +1,113 @@ +--- +display_name: Claude Code +description: Run Claude Code in your workspace +icon: ../.icons/claude.svg +verified: true +tags: [agent, claude-code] +--- + +# Claude Code + +Run the [Claude Code](https://docs.anthropic.com/en/docs/agents-and-tools/claude-code/overview) agent in your workspace to generate code and perform tasks. + +```tf +module "claude-code" { + source = "registry.coder.com/modules/claude-code/coder" + version = "1.0.31" + agent_id = coder_agent.example.id + folder = "/home/coder" + install_claude_code = true + claude_code_version = "latest" +} +``` + +## Prerequisites + +- Node.js and npm must be installed in your workspace to install Claude Code +- `screen` must be installed in your workspace to run Claude Code in the background +- You must add the [Coder Login](https://registry.coder.com/modules/coder-login) module to your template + +The `codercom/oss-dogfood:latest` container image can be used for testing on container-based workspaces. + +## Examples + +### Run in the background and report tasks (Experimental) + +> This functionality is in early access as of Coder v2.21 and is still evolving. +> For now, we recommend testing it in a demo or staging environment, +> rather than deploying to production +> +> Learn more in [the Coder documentation](https://coder.com/docs/tutorials/ai-agents) +> +> Join our [Discord channel](https://discord.gg/coder) or +> [contact us](https://coder.com/contact) to get help or share feedback. + +Your workspace must have `screen` installed to use this. + +```tf +variable "anthropic_api_key" { + type = string + description = "The Anthropic API key" + sensitive = true +} + +module "coder-login" { + count = data.coder_workspace.me.start_count + source = "registry.coder.com/modules/coder-login/coder" + version = "1.0.15" + agent_id = coder_agent.example.id +} + +data "coder_parameter" "ai_prompt" { + type = "string" + name = "AI Prompt" + default = "" + description = "Write a prompt for Claude Code" + mutable = true +} + +# Set the prompt and system prompt for Claude Code via environment variables +resource "coder_agent" "main" { + # ... + env = { + CODER_MCP_CLAUDE_API_KEY = var.anthropic_api_key # or use a coder_parameter + CODER_MCP_CLAUDE_TASK_PROMPT = data.coder_parameter.ai_prompt.value + CODER_MCP_APP_STATUS_SLUG = "claude-code" + CODER_MCP_CLAUDE_SYSTEM_PROMPT = <<-EOT + You are a helpful assistant that can help with code. + EOT + } +} + +module "claude-code" { + count = data.coder_workspace.me.start_count + source = "registry.coder.com/modules/claude-code/coder" + version = "1.0.31" + agent_id = coder_agent.example.id + folder = "/home/coder" + install_claude_code = true + claude_code_version = "0.2.57" + + # Enable experimental features + experiment_use_screen = true + experiment_report_tasks = true +} +``` + +## Run standalone + +Run Claude Code as a standalone app in your workspace. This will install Claude Code and run it directly without using screen or any task reporting to the Coder UI. + +```tf +module "claude-code" { + source = "registry.coder.com/modules/claude-code/coder" + version = "1.0.31" + agent_id = coder_agent.example.id + folder = "/home/coder" + install_claude_code = true + claude_code_version = "latest" + + # Icon is not available in Coder v2.20 and below, so we'll use a custom icon URL + icon = "https://registry.npmmirror.com/@lobehub/icons-static-png/1.24.0/files/dark/claude-color.png" +} +``` diff --git a/registry/coder/modules/claude-code/main.tf b/registry/coder/modules/claude-code/main.tf new file mode 100644 index 0000000..349af17 --- /dev/null +++ b/registry/coder/modules/claude-code/main.tf @@ -0,0 +1,170 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + coder = { + source = "coder/coder" + version = ">= 0.17" + } + } +} + +variable "agent_id" { + type = string + description = "The ID of a Coder agent." +} + +data "coder_workspace" "me" {} + +data "coder_workspace_owner" "me" {} + +variable "order" { + type = number + description = "The order determines the position of app in the UI presentation. The lowest order is shown first and apps with equal order are sorted by name (ascending order)." + default = null +} + +variable "icon" { + type = string + description = "The icon to use for the app." + default = "/icon/claude.svg" +} + +variable "folder" { + type = string + description = "The folder to run Claude Code in." + default = "/home/coder" +} + +variable "install_claude_code" { + type = bool + description = "Whether to install Claude Code." + default = true +} + +variable "claude_code_version" { + type = string + description = "The version of Claude Code to install." + default = "latest" +} + +variable "experiment_use_screen" { + type = bool + description = "Whether to use screen for running Claude Code in the background." + default = false +} + +variable "experiment_report_tasks" { + type = bool + description = "Whether to enable task reporting." + default = false +} + +# Install and Initialize Claude Code +resource "coder_script" "claude_code" { + agent_id = var.agent_id + display_name = "Claude Code" + icon = var.icon + script = <<-EOT + #!/bin/bash + set -e + + # Function to check if a command exists + command_exists() { + command -v "$1" >/dev/null 2>&1 + } + + # Install Claude Code if enabled + if [ "${var.install_claude_code}" = "true" ]; then + if ! command_exists npm; then + echo "Error: npm is not installed. Please install Node.js and npm first." + exit 1 + fi + echo "Installing Claude Code..." + npm install -g @anthropic-ai/claude-code@${var.claude_code_version} + fi + + if [ "${var.experiment_report_tasks}" = "true" ]; then + echo "Configuring Claude Code to report tasks via Coder MCP..." + coder exp mcp configure claude-code ${var.folder} + fi + + # Run with screen if enabled + if [ "${var.experiment_use_screen}" = "true" ]; then + echo "Running Claude Code in the background..." + + # Check if screen is installed + if ! command_exists screen; then + echo "Error: screen is not installed. Please install screen manually." + exit 1 + fi + + touch "$HOME/.claude-code.log" + + # Ensure the screenrc exists + if [ ! -f "$HOME/.screenrc" ]; then + echo "Creating ~/.screenrc and adding multiuser settings..." | tee -a "$HOME/.claude-code.log" + echo -e "multiuser on\nacladd $(whoami)" > "$HOME/.screenrc" + fi + + if ! grep -q "^multiuser on$" "$HOME/.screenrc"; then + echo "Adding 'multiuser on' to ~/.screenrc..." | tee -a "$HOME/.claude-code.log" + echo "multiuser on" >> "$HOME/.screenrc" + fi + + if ! grep -q "^acladd $(whoami)$" "$HOME/.screenrc"; then + echo "Adding 'acladd $(whoami)' to ~/.screenrc..." | tee -a "$HOME/.claude-code.log" + echo "acladd $(whoami)" >> "$HOME/.screenrc" + fi + export LANG=en_US.UTF-8 + export LC_ALL=en_US.UTF-8 + + screen -U -dmS claude-code bash -c ' + cd ${var.folder} + claude --dangerously-skip-permissions | tee -a "$HOME/.claude-code.log" + exec bash + ' + # Extremely hacky way to send the prompt to the screen session + # This will be fixed in the future, but `claude` was not sending MCP + # tasks when an initial prompt is provided. + screen -S claude-code -X stuff "$CODER_MCP_CLAUDE_TASK_PROMPT" + sleep 5 + screen -S claude-code -X stuff "^M" + else + # Check if claude is installed before running + if ! command_exists claude; then + echo "Error: Claude Code is not installed. Please enable install_claude_code or install it manually." + exit 1 + fi + fi + EOT + run_on_start = true +} + +resource "coder_app" "claude_code" { + slug = "claude-code" + display_name = "Claude Code" + agent_id = var.agent_id + command = <<-EOT + #!/bin/bash + set -e + + if [ "${var.experiment_use_screen}" = "true" ]; then + if screen -list | grep -q "claude-code"; then + export LANG=en_US.UTF-8 + export LC_ALL=en_US.UTF-8 + echo "Attaching to existing Claude Code session." | tee -a "$HOME/.claude-code.log" + screen -xRR claude-code + else + echo "Starting a new Claude Code session." | tee -a "$HOME/.claude-code.log" + screen -S claude-code bash -c 'export LANG=en_US.UTF-8; export LC_ALL=en_US.UTF-8; claude --dangerously-skip-permissions | tee -a "$HOME/.claude-code.log"; exec bash' + fi + else + cd ${var.folder} + export LANG=en_US.UTF-8 + export LC_ALL=en_US.UTF-8 + claude + fi + EOT + icon = var.icon +} diff --git a/registry/coder/modules/cursor/README.md b/registry/coder/modules/cursor/README.md new file mode 100644 index 0000000..691526c --- /dev/null +++ b/registry/coder/modules/cursor/README.md @@ -0,0 +1,36 @@ +--- +display_name: Cursor IDE +description: Add a one-click button to launch Cursor IDE +icon: ../.icons/cursor.svg +verified: true +tags: [ide, cursor, helper] +--- + +# Cursor IDE + +Add a button to open any workspace with a single click in Cursor IDE. + +Uses the [Coder Remote VS Code Extension](https://github.com/coder/vscode-coder). + +```tf +module "cursor" { + count = data.coder_workspace.me.start_count + source = "registry.coder.com/modules/cursor/coder" + version = "1.0.19" + agent_id = coder_agent.example.id +} +``` + +## Examples + +### Open in a specific directory + +```tf +module "cursor" { + count = data.coder_workspace.me.start_count + source = "registry.coder.com/modules/cursor/coder" + version = "1.0.19" + agent_id = coder_agent.example.id + folder = "/home/coder/project" +} +``` diff --git a/registry/coder/modules/cursor/main.test.ts b/registry/coder/modules/cursor/main.test.ts new file mode 100644 index 0000000..3c16469 --- /dev/null +++ b/registry/coder/modules/cursor/main.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "bun:test"; +import { + runTerraformApply, + runTerraformInit, + testRequiredVariables, +} from "../test"; + +describe("cursor", async () => { + await runTerraformInit(import.meta.dir); + + testRequiredVariables(import.meta.dir, { + agent_id: "foo", + }); + + it("default output", async () => { + const state = await runTerraformApply(import.meta.dir, { + agent_id: "foo", + }); + expect(state.outputs.cursor_url.value).toBe( + "cursor://coder.coder-remote/open?owner=default&workspace=default&url=https://mydeployment.coder.com&token=$SESSION_TOKEN", + ); + + const coder_app = state.resources.find( + (res) => res.type === "coder_app" && res.name === "cursor", + ); + + expect(coder_app).not.toBeNull(); + expect(coder_app?.instances.length).toBe(1); + expect(coder_app?.instances[0].attributes.order).toBeNull(); + }); + + it("adds folder", async () => { + const state = await runTerraformApply(import.meta.dir, { + agent_id: "foo", + folder: "/foo/bar", + }); + expect(state.outputs.cursor_url.value).toBe( + "cursor://coder.coder-remote/open?owner=default&workspace=default&folder=/foo/bar&url=https://mydeployment.coder.com&token=$SESSION_TOKEN", + ); + }); + + it("adds folder and open_recent", async () => { + const state = await runTerraformApply(import.meta.dir, { + agent_id: "foo", + folder: "/foo/bar", + open_recent: "true", + }); + expect(state.outputs.cursor_url.value).toBe( + "cursor://coder.coder-remote/open?owner=default&workspace=default&folder=/foo/bar&openRecent&url=https://mydeployment.coder.com&token=$SESSION_TOKEN", + ); + }); + + it("adds folder but not open_recent", async () => { + const state = await runTerraformApply(import.meta.dir, { + agent_id: "foo", + folder: "/foo/bar", + openRecent: "false", + }); + expect(state.outputs.cursor_url.value).toBe( + "cursor://coder.coder-remote/open?owner=default&workspace=default&folder=/foo/bar&url=https://mydeployment.coder.com&token=$SESSION_TOKEN", + ); + }); + + it("adds open_recent", async () => { + const state = await runTerraformApply(import.meta.dir, { + agent_id: "foo", + open_recent: "true", + }); + expect(state.outputs.cursor_url.value).toBe( + "cursor://coder.coder-remote/open?owner=default&workspace=default&openRecent&url=https://mydeployment.coder.com&token=$SESSION_TOKEN", + ); + }); + + it("expect order to be set", async () => { + const state = await runTerraformApply(import.meta.dir, { + agent_id: "foo", + order: "22", + }); + + const coder_app = state.resources.find( + (res) => res.type === "coder_app" && res.name === "cursor", + ); + + expect(coder_app).not.toBeNull(); + expect(coder_app?.instances.length).toBe(1); + expect(coder_app?.instances[0].attributes.order).toBe(22); + }); +}); diff --git a/registry/coder/modules/cursor/main.tf b/registry/coder/modules/cursor/main.tf new file mode 100644 index 0000000..f350f94 --- /dev/null +++ b/registry/coder/modules/cursor/main.tf @@ -0,0 +1,62 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + coder = { + source = "coder/coder" + version = ">= 0.23" + } + } +} + +variable "agent_id" { + type = string + description = "The ID of a Coder agent." +} + +variable "folder" { + type = string + description = "The folder to open in Cursor IDE." + default = "" +} + +variable "open_recent" { + type = bool + description = "Open the most recent workspace or folder. Falls back to the folder if there is no recent workspace or folder to open." + default = false +} + +variable "order" { + type = number + description = "The order determines the position of app in the UI presentation. The lowest order is shown first and apps with equal order are sorted by name (ascending order)." + default = null +} + +data "coder_workspace" "me" {} +data "coder_workspace_owner" "me" {} + +resource "coder_app" "cursor" { + agent_id = var.agent_id + external = true + icon = "/icon/cursor.svg" + slug = "cursor" + display_name = "Cursor Desktop" + order = var.order + url = join("", [ + "cursor://coder.coder-remote/open", + "?owner=", + data.coder_workspace_owner.me.name, + "&workspace=", + data.coder_workspace.me.name, + var.folder != "" ? join("", ["&folder=", var.folder]) : "", + var.open_recent ? "&openRecent" : "", + "&url=", + data.coder_workspace.me.access_url, + "&token=$SESSION_TOKEN", + ]) +} + +output "cursor_url" { + value = coder_app.cursor.url + description = "Cursor IDE Desktop URL." +} From e6efd71fcafa6c442ca134f88ac0a1bfc1ff59c5 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Wed, 16 Apr 2025 04:02:45 +0000 Subject: [PATCH 22/23] wip: more progress --- cmd/readmevalidation/coderResources.go | 74 +++++++++++++++++++++++- cmd/readmevalidation/main.go | 78 +++++++++++++------------- cmd/readmevalidation/readmes.go | 45 +++++++++++++++ cmd/readmevalidation/repoStructure.go | 27 +++++---- 4 files changed, 170 insertions(+), 54 deletions(-) diff --git a/cmd/readmevalidation/coderResources.go b/cmd/readmevalidation/coderResources.go index eda5fd6..d4661c5 100644 --- a/cmd/readmevalidation/coderResources.go +++ b/cmd/readmevalidation/coderResources.go @@ -10,6 +10,7 @@ import ( "strings" "coder.com/coder-registry/cmd/github" + "gopkg.in/yaml.v3" ) // dummyGitDirectory is the directory that a full version of the Registry will @@ -34,7 +35,7 @@ type coderResourceFrontmatter struct { // Coder Modules and Coder Templates. If the newReadmeBody and newFrontmatter // fields are nil, that represents that the file has been deleted type coderResource struct { - name string + resourceType string filePath string newReadmeBody *string oldFrontmatter *coderResourceFrontmatter @@ -178,10 +179,77 @@ func validateCoderResourceChanges(resource coderResource, actorOrgStatus github. return problems } -func parseCoderResourceFiles(oldReadmeFiles []readme, newReadmeFiles []readme, actorOrgStatus github.OrgStatus) (map[string]coderResource, error) { - return nil, nil +func parseCoderResourceFiles(resourceType string, oldReadmeFiles []readme, newReadmeFiles []readme, actorOrgStatus github.OrgStatus) (map[string]coderResource, error) { + if !slices.Contains(supportedResourceTypes, resourceType) { + return nil, fmt.Errorf("resource type %q is not in supported list [%s]", resourceType, strings.Join(supportedResourceTypes, ", ")) + } + + var errs []error + resourcesByFilePath := map[string]coderResource{} + zipped := zipReadmes(oldReadmeFiles, newReadmeFiles) + + for filePath, z := range zipped { + resource := coderResource{ + resourceType: resourceType, + filePath: filePath, + } + + if z.new != nil { + fm, body, err := separateFrontmatter(z.new.rawText) + if err != nil { + errs = append(errs, fmt.Errorf("resource type %s - %q: %v", resourceType, filePath, err)) + } else { + resource.newReadmeBody = &body + var newFm coderResourceFrontmatter + if err := yaml.Unmarshal([]byte(fm), &newFm); err != nil { + errs = append(errs, fmt.Errorf("resource type %s - %q: %v", resourceType, filePath, err)) + } else { + resource.newFrontmatter = &newFm + if newFm.Verified != nil && *newFm.Verified { + resource.newIsVerified = true + } + } + } + } + + if z.old != nil { + fm, _, err := separateFrontmatter(z.old.rawText) + if err != nil { + errs = append(errs, fmt.Errorf("resource type %s - %q: %v", resourceType, filePath, err)) + } else { + var oldFm coderResourceFrontmatter + if err := yaml.Unmarshal([]byte(fm), &oldFm); err != nil { + errs = append(errs, fmt.Errorf("resource type %s - %q: %v", resourceType, filePath, err)) + } else { + resource.oldFrontmatter = &oldFm + if oldFm.Verified != nil && *oldFm.Verified { + resource.oldIsVerified = true + } + } + } + } + + if z.old != nil || z.new != nil { + resourcesByFilePath[filePath] = resource + } + } + + for _, r := range resourcesByFilePath { + errs = append(errs, validateCoderResourceChanges(r, actorOrgStatus)...) + } + + if len(errs) != 0 { + return nil, validationPhaseError{ + phase: validationPhaseReadmeParsing, + errors: errs, + } + } + return resourcesByFilePath, nil } +// Todo: because Coder Resource READMEs will have their full contents +// (frontmatter and body) rendered on the Registry site, we need to make sure +// that all image references in the body are valid, too func validateCoderResourceRelativeUrls(map[string]coderResource) []error { return nil } diff --git a/cmd/readmevalidation/main.go b/cmd/readmevalidation/main.go index f58fc47..56ee382 100644 --- a/cmd/readmevalidation/main.go +++ b/cmd/readmevalidation/main.go @@ -13,9 +13,6 @@ import ( "sync" "coder.com/coder-registry/cmd/github" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/joho/godotenv" ) @@ -104,53 +101,56 @@ func main() { go func() { defer wg.Done() - refactorLater := func() error { + moveToOuterScopeLater := func() error { baseRefReadmeFiles, err := aggregateCoderResourceReadmeFiles("modules") if err != nil { return err } - fmt.Printf("------ got %d back\n", len(baseRefReadmeFiles)) - - repo, err := git.PlainClone(dummyGitDirectory, true, &git.CloneOptions{ - URL: "https://github.com/coder/registry", - Auth: &http.BasicAuth{}, - }) - if err != nil { - return err - } - - head, err := repo.Head() - if err != nil { - return err - } - activeBranchName := head.Name().Short() - - tree, err := repo.Worktree() + parsed, err := parseCoderResourceFiles("modules", baseRefReadmeFiles, baseRefReadmeFiles, actorOrgStatus) if err != nil { return err } - err = tree.Checkout(&git.CheckoutOptions{ - Branch: plumbing.NewBranchReferenceName(activeBranchName), - Create: false, - Force: false, - Keep: true, - }) - if err != nil { - return err - } - - fmt.Println("Got here!") - files, _ := tree.Filesystem.ReadDir(".") - for _, f := range files { - if f.IsDir() { - fmt.Println(f.Name()) - } - } + fmt.Printf("------ got %d back\n", len(parsed)) + + // repo, err := git.PlainClone(dummyGitDirectory, true, &git.CloneOptions{ + // URL: "https://github.com/coder/registry", + // Auth: &http.BasicAuth{}, + // }) + // if err != nil { + // return err + // } + + // head, err := repo.Head() + // if err != nil { + // return err + // } + // activeBranchName := head.Name().Short() + + // tree, err := repo.Worktree() + // if err != nil { + // return err + // } + // err = tree.Checkout(&git.CheckoutOptions{ + // Branch: plumbing.NewBranchReferenceName(activeBranchName), + // Create: false, + // Force: false, + // Keep: true, + // }) + // if err != nil { + // return err + // } + + // files, _ := tree.Filesystem.ReadDir(".") + // for _, f := range files { + // if f.IsDir() { + // fmt.Println(f.Name()) + // } + // } return nil } - if err := refactorLater(); err != nil { + if err := moveToOuterScopeLater(); err != nil { errChan <- fmt.Errorf("module validation: %v", err) } diff --git a/cmd/readmevalidation/readmes.go b/cmd/readmevalidation/readmes.go index 00bcca1..78de0e4 100644 --- a/cmd/readmevalidation/readmes.go +++ b/cmd/readmevalidation/readmes.go @@ -108,3 +108,48 @@ func (p validationPhase) String() string { return "Unknown validation phase" } } + +type zippedReadmes struct { + old *readme + new *readme +} + +// zipReadmes takes two slices of README files, and combines them into a map, +// where each key is a file path, and each value is a struct containing the old +// value for the path, and the new value for the path. If the old value exists +// but the new one doesn't, that indicates that a file has been deleted. If the +// new value exists, but the old one doesn't, that indicates that the file was +// created. +func zipReadmes(prevReadmes []readme, newReadmes []readme) map[string]zippedReadmes { + oldMap := map[string]readme{} + for _, rm := range prevReadmes { + oldMap[rm.filePath] = rm + } + + zipped := map[string]zippedReadmes{} + for _, rm := range newReadmes { + old, ok := oldMap[rm.filePath] + if ok { + zipped[rm.filePath] = zippedReadmes{ + old: &old, + new: &rm, + } + } else { + zipped[rm.filePath] = zippedReadmes{ + old: nil, + new: &rm, + } + } + } + for _, old := range oldMap { + _, ok := zipped[old.filePath] + if !ok { + zipped[old.filePath] = zippedReadmes{ + old: &old, + new: nil, + } + } + } + + return zipped +} diff --git a/cmd/readmevalidation/repoStructure.go b/cmd/readmevalidation/repoStructure.go index ebc8809..fe1d905 100644 --- a/cmd/readmevalidation/repoStructure.go +++ b/cmd/readmevalidation/repoStructure.go @@ -37,15 +37,24 @@ func validateCoderResourceSubdirectory(dirPath string) []error { resourceReadmePath := path.Join(dirPath, f.Name(), "README.md") _, err := os.Stat(resourceReadmePath) - if err == nil { - continue + if err != nil { + if errors.Is(err, os.ErrNotExist) { + errs = append(errs, fmt.Errorf("%q: README file does not exist", resourceReadmePath)) + } else { + errs = append(errs, addFilePathToError(resourceReadmePath, err)) + } } - if errors.Is(err, os.ErrNotExist) { - errs = append(errs, fmt.Errorf("%q: README file does not exist", resourceReadmePath)) - } else { - errs = append(errs, addFilePathToError(resourceReadmePath, err)) + mainTerraformPath := path.Join(dirPath, f.Name(), "main.tf") + _, err = os.Stat(mainTerraformPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + errs = append(errs, fmt.Errorf("%q: 'main.tf' file does not exist", mainTerraformPath)) + } else { + errs = append(errs, addFilePathToError(mainTerraformPath, err)) + } } + } return errs @@ -71,12 +80,6 @@ func validateRegistryDirectory() []error { problems = append(problems, err) } - mainTerraformPath := path.Join(dirPath, "main.tf") - _, err = os.Stat(mainTerraformPath) - if err != nil { - problems = append(problems, err) - } - for _, rType := range supportedResourceTypes { resourcePath := path.Join(dirPath, rType) if errs := validateCoderResourceSubdirectory(resourcePath); len(errs) != 0 { From d2ebc2b1d9e5b2c4c140ad72693bbbb0fc2082ca Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Wed, 16 Apr 2025 15:22:13 +0000 Subject: [PATCH 23/23] fix: update icon URLs --- cmd/readmevalidation/coderResources.go | 4 ++-- cmd/readmevalidation/main.go | 4 ++-- cmd/readmevalidation/repoStructure.go | 2 +- registry/coder/modules/claude-code/README.md | 3 ++- registry/coder/modules/cursor/README.md | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cmd/readmevalidation/coderResources.go b/cmd/readmevalidation/coderResources.go index d4661c5..57b4407 100644 --- a/cmd/readmevalidation/coderResources.go +++ b/cmd/readmevalidation/coderResources.go @@ -87,9 +87,9 @@ func validateCoderResourceIconURL(iconURL string) []error { // repo, and where this logic will run isPermittedRelativeURL := strings.HasPrefix(iconURL, "./") || strings.HasPrefix(iconURL, "/") || - strings.HasPrefix(iconURL, "../../../.logos") + strings.HasPrefix(iconURL, "../../../.icons") if !isPermittedRelativeURL { - problems = append(problems, errors.New("relative icon URL must either be scoped to that module's directory, or the top-level /.logos directory")) + problems = append(problems, fmt.Errorf("relative icon URL %q must either be scoped to that module's directory, or the top-level /.icons directory (this can usually be done by starting the path with \"../../../.icons\")", iconURL)) } return problems diff --git a/cmd/readmevalidation/main.go b/cmd/readmevalidation/main.go index 56ee382..5abef39 100644 --- a/cmd/readmevalidation/main.go +++ b/cmd/readmevalidation/main.go @@ -172,9 +172,9 @@ func main() { } fmt.Println("---") - fmt.Println("Encountered the following problems") + log.Println("Encountered the following problems") for _, err := range readmeValidationErrors { - log.Println(err) + fmt.Println(err) } os.Exit(1) } diff --git a/cmd/readmevalidation/repoStructure.go b/cmd/readmevalidation/repoStructure.go index fe1d905..951abde 100644 --- a/cmd/readmevalidation/repoStructure.go +++ b/cmd/readmevalidation/repoStructure.go @@ -97,7 +97,7 @@ func validateRepoStructure() error { problems = append(problems, errs...) } - _, err := os.Stat("./.logos") + _, err := os.Stat("./.icons") if err != nil { problems = append(problems, err) } diff --git a/registry/coder/modules/claude-code/README.md b/registry/coder/modules/claude-code/README.md index 2cb506b..01cb5a6 100644 --- a/registry/coder/modules/claude-code/README.md +++ b/registry/coder/modules/claude-code/README.md @@ -1,7 +1,8 @@ --- display_name: Claude Code description: Run Claude Code in your workspace -icon: ../.icons/claude.svg +icon: ../../../.icons/claude.svg +maintainer_github: coder verified: true tags: [agent, claude-code] --- diff --git a/registry/coder/modules/cursor/README.md b/registry/coder/modules/cursor/README.md index 691526c..b6d9815 100644 --- a/registry/coder/modules/cursor/README.md +++ b/registry/coder/modules/cursor/README.md @@ -1,7 +1,7 @@ --- display_name: Cursor IDE description: Add a one-click button to launch Cursor IDE -icon: ../.icons/cursor.svg +icon: ../../../.icons/cursor.svg verified: true tags: [ide, cursor, helper] ---