From 7bc91f20fb82eefdf3d0b9d1a175a5136154b8f8 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Tue, 21 Nov 2023 11:29:13 -0600 Subject: [PATCH 1/6] chore: add script to analyze which releases have migrations --- scripts/releasemigrations/main.go | 211 ++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 scripts/releasemigrations/main.go diff --git a/scripts/releasemigrations/main.go b/scripts/releasemigrations/main.go new file mode 100644 index 0000000000000..d4de035cc885f --- /dev/null +++ b/scripts/releasemigrations/main.go @@ -0,0 +1,211 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os/exec" + "strings" + + "golang.org/x/xerrors" + + "golang.org/x/mod/semver" +) + +// main will print out the number of migrations added between each release. +// All upgrades are categorized as either major, minor, or patch based on semver. +// +// This isn't an exact science and is opinionated. Upgrade paths are not +// always strictly linear from release to release. Users can skip patches for +// example. +func main() { + var includePatches bool + var includeMinors bool + var includeMajors bool + // If you only run with --patches, the upgrades that are minors are excluded. + // Example being 1.0.0 -> 1.1.0 is a minor upgrade, so it's not included. + flag.BoolVar(&includePatches, "patches", false, "Include patches releases") + flag.BoolVar(&includeMinors, "minors", false, "Include minor releases") + flag.BoolVar(&includeMajors, "majors", false, "Include major releases") + flag.Parse() + + if !includePatches && !includeMinors && !includeMajors { + usage() + return + } + + err := run(Options{ + IncludePatches: includePatches, + IncludeMinors: includeMinors, + IncludeMajors: includeMajors, + }) + if err != nil { + log.Fatal(err) + } +} + +func usage() { + fmt.Println("Usage: releasemigrations [--patches] [--minors] [--majors]") + fmt.Println("Choose at lease one of --patches, --minors, or --majors. You can choose all!") +} + +type Options struct { + IncludePatches bool + IncludeMinors bool + IncludeMajors bool +} + +func (o Options) Filter(tags []string) []string { + if o.IncludeMajors && o.IncludeMinors && o.IncludePatches { + return tags + } + + filtered := make([]string, 0, len(tags)) + current := tags[0] + filtered = append(filtered, current) + for i := 1; i < len(tags); i++ { + a := current + current = tags[i] + + vDiffType := versionDiff(a, tags[i]) + if !o.IncludeMajors && vDiffType == "major" { + continue + } + if !o.IncludeMinors && vDiffType == "minor" { + // This isn't perfect, but we need to include + // the first minor release for the first patch to work. + // Eg: 1.0.0 -> 1.1.0 -> 1.1.1 + // If we didn't include 1.1.0, then the 1.1.1 patch would + // apply to 1.0.0 + if !o.IncludePatches { + continue + } + } + if !o.IncludePatches && vDiffType == "patch" { + continue + } + filtered = append(filtered, tags[i]) + } + + return filtered +} + +func run(opts Options) error { + tags, err := gitTags() + if err != nil { + return xerrors.Errorf("gitTags: %w", err) + } + tags = opts.Filter(tags) + + patches := make([]string, 0) + minors := make([]string, 0) + majors := make([]string, 0) + patchesHasMig := 0 + minorsHasMig := 0 + majorsHasMig := 0 + + for i := 0; i < len(tags)-1; i++ { + a := tags[i] + b := tags[i+1] + + migrations, err := hasMigrationDiff(a, b) + if err != nil { + return xerrors.Errorf("hasMigrationDiff %q->%q: %w", a, b, err) + } + + vDiff := fmt.Sprintf("%s->%s", a, b) + vDiffType := versionDiff(a, b) + skipPrint := true + switch vDiffType { + case "major": + majors = append(majors, vDiff) + if len(migrations) > 0 { + majorsHasMig++ + } + skipPrint = !opts.IncludeMajors + case "minor": + minors = append(minors, vDiff) + if len(migrations) > 0 { + minorsHasMig++ + } + skipPrint = !opts.IncludeMinors + case "patch": + patches = append(patches, vDiff) + if len(migrations) > 0 { + patchesHasMig++ + } + skipPrint = !opts.IncludePatches + } + + if skipPrint { + continue + } + + if migrations != nil { + log.Printf("[%s] %d migrations added between %s and %s\n", vDiffType, len(migrations)/2, a, b) + //for _, migration := range migrations { + // log.Printf(" %s\n", migration) + //} + } else { + log.Printf("[%s] No migrations added between %s and %s\n", vDiffType, a, b) + } + } + + log.Printf("Patches: %d (%d with migrations)\n", len(patches), patchesHasMig) + log.Printf("Minors: %d (%d with migrations)\n", len(minors), minorsHasMig) + log.Printf("Majors: %d (%d with migrations)\n", len(majors), majorsHasMig) + + return nil +} + +func versionDiff(a, b string) string { + ac, bc := semver.Canonical(a), semver.Canonical(b) + if semver.Major(ac) != semver.Major(bc) { + return "major" + } + if semver.MajorMinor(ac) != semver.MajorMinor(bc) { + return "minor" + } + return "patch" +} + +func hasMigrationDiff(a, b string) ([]string, error) { + cmd := exec.Command("git", "diff", + // Only added files + "--diff-filter=A", + "--name-only", + a, b, "coderd/database/migrations") + output, err := cmd.Output() + if err != nil { + return nil, xerrors.Errorf("%s\n%s", strings.Join(cmd.Args, " "), err) + return nil, err + } + if len(output) == 0 { + return nil, nil + } + + migrations := strings.Split(string(output), "\n") + return migrations, nil +} + +func gitTags() ([]string, error) { + cmd := exec.Command("git", "tag") + output, err := cmd.Output() + if err != nil { + return nil, err + } + + tags := strings.Split(string(output), "\n") + + // Sort by semver + semver.Sort(tags) + + filtered := make([]string, 0, len(tags)) + for _, tag := range tags { + if tag != "" && semver.IsValid(tag) { + filtered = append(filtered, tag) + } + } + + return filtered, nil +} From 073b641600da16ce56609a8a05a2aac137bb1187 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Tue, 21 Nov 2023 11:32:20 -0600 Subject: [PATCH 2/6] add v2 flag --- scripts/releasemigrations/main.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/releasemigrations/main.go b/scripts/releasemigrations/main.go index d4de035cc885f..0794230af78a6 100644 --- a/scripts/releasemigrations/main.go +++ b/scripts/releasemigrations/main.go @@ -22,11 +22,13 @@ func main() { var includePatches bool var includeMinors bool var includeMajors bool + var afterV2 bool // If you only run with --patches, the upgrades that are minors are excluded. // Example being 1.0.0 -> 1.1.0 is a minor upgrade, so it's not included. flag.BoolVar(&includePatches, "patches", false, "Include patches releases") flag.BoolVar(&includeMinors, "minors", false, "Include minor releases") flag.BoolVar(&includeMajors, "majors", false, "Include major releases") + flag.BoolVar(&afterV2, "after-v2", false, "Only include releases after v2.0.0") flag.Parse() if !includePatches && !includeMinors && !includeMajors { @@ -38,6 +40,7 @@ func main() { IncludePatches: includePatches, IncludeMinors: includeMinors, IncludeMajors: includeMajors, + AfterV2: afterV2, }) if err != nil { log.Fatal(err) @@ -53,9 +56,19 @@ type Options struct { IncludePatches bool IncludeMinors bool IncludeMajors bool + AfterV2 bool } func (o Options) Filter(tags []string) []string { + if o.AfterV2 { + for i, tag := range tags { + if tag == "v2.0.0" { + tags = tags[i:] + break + } + } + } + if o.IncludeMajors && o.IncludeMinors && o.IncludePatches { return tags } From 6422921e8b324991143f0512ea2eec41dc7723dd Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Tue, 21 Nov 2023 11:36:01 -0600 Subject: [PATCH 3/6] Add listnig migrations command --- scripts/releasemigrations/main.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/releasemigrations/main.go b/scripts/releasemigrations/main.go index 0794230af78a6..ab061131c1d17 100644 --- a/scripts/releasemigrations/main.go +++ b/scripts/releasemigrations/main.go @@ -23,12 +23,14 @@ func main() { var includeMinors bool var includeMajors bool var afterV2 bool + var listMigs bool // If you only run with --patches, the upgrades that are minors are excluded. // Example being 1.0.0 -> 1.1.0 is a minor upgrade, so it's not included. flag.BoolVar(&includePatches, "patches", false, "Include patches releases") flag.BoolVar(&includeMinors, "minors", false, "Include minor releases") flag.BoolVar(&includeMajors, "majors", false, "Include major releases") flag.BoolVar(&afterV2, "after-v2", false, "Only include releases after v2.0.0") + flag.BoolVar(&listMigs, "list", false, "List migrations") flag.Parse() if !includePatches && !includeMinors && !includeMajors { @@ -41,6 +43,7 @@ func main() { IncludeMinors: includeMinors, IncludeMajors: includeMajors, AfterV2: afterV2, + ListMigrations: listMigs, }) if err != nil { log.Fatal(err) @@ -57,6 +60,7 @@ type Options struct { IncludeMinors bool IncludeMajors bool AfterV2 bool + ListMigrations bool } func (o Options) Filter(tags []string) []string { @@ -156,9 +160,11 @@ func run(opts Options) error { if migrations != nil { log.Printf("[%s] %d migrations added between %s and %s\n", vDiffType, len(migrations)/2, a, b) - //for _, migration := range migrations { - // log.Printf(" %s\n", migration) - //} + if opts.ListMigrations { + for _, migration := range migrations { + log.Printf("\t%s", migration) + } + } } else { log.Printf("[%s] No migrations added between %s and %s\n", vDiffType, a, b) } @@ -191,13 +197,12 @@ func hasMigrationDiff(a, b string) ([]string, error) { output, err := cmd.Output() if err != nil { return nil, xerrors.Errorf("%s\n%s", strings.Join(cmd.Args, " "), err) - return nil, err } if len(output) == 0 { return nil, nil } - migrations := strings.Split(string(output), "\n") + migrations := strings.Split(strings.TrimSpace(string(output)), "\n") return migrations, nil } From 2121d8d28573f418ba18cd7de2c88cff4e81082d Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Tue, 21 Nov 2023 11:52:28 -0600 Subject: [PATCH 4/6] Reduce some output noise by only showing ups --- scripts/releasemigrations/main.go | 77 +++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 20 deletions(-) diff --git a/scripts/releasemigrations/main.go b/scripts/releasemigrations/main.go index ab061131c1d17..545618db1f545 100644 --- a/scripts/releasemigrations/main.go +++ b/scripts/releasemigrations/main.go @@ -24,26 +24,42 @@ func main() { var includeMajors bool var afterV2 bool var listMigs bool + var migrationDirectory string + var versionList string + // If you only run with --patches, the upgrades that are minors are excluded. // Example being 1.0.0 -> 1.1.0 is a minor upgrade, so it's not included. flag.BoolVar(&includePatches, "patches", false, "Include patches releases") flag.BoolVar(&includeMinors, "minors", false, "Include minor releases") flag.BoolVar(&includeMajors, "majors", false, "Include major releases") + flag.StringVar(&versionList, "versions", "", "Comma separated list of versions to use. This skips uses git tag to find tags.") flag.BoolVar(&afterV2, "after-v2", false, "Only include releases after v2.0.0") flag.BoolVar(&listMigs, "list", false, "List migrations") + flag.StringVar(&migrationDirectory, "dir", "coderd/database/migrations", "Migration directory") flag.Parse() - if !includePatches && !includeMinors && !includeMajors { + if !includePatches && !includeMinors && !includeMajors && versionList == "" { usage() return } + var vList []string + if versionList != "" { + // Include all for printing purposes. + includeMajors = true + includeMinors = true + includePatches = true + vList = strings.Split(versionList, ",") + } + err := run(Options{ - IncludePatches: includePatches, - IncludeMinors: includeMinors, - IncludeMajors: includeMajors, - AfterV2: afterV2, - ListMigrations: listMigs, + VersionList: vList, + IncludePatches: includePatches, + IncludeMinors: includeMinors, + IncludeMajors: includeMajors, + AfterV2: afterV2, + ListMigrations: listMigs, + MigrationDirectory: migrationDirectory, }) if err != nil { log.Fatal(err) @@ -53,14 +69,17 @@ func main() { func usage() { fmt.Println("Usage: releasemigrations [--patches] [--minors] [--majors]") fmt.Println("Choose at lease one of --patches, --minors, or --majors. You can choose all!") + fmt.Println("Must be run from the coder repo at the root.") } type Options struct { - IncludePatches bool - IncludeMinors bool - IncludeMajors bool - AfterV2 bool - ListMigrations bool + VersionList []string + IncludePatches bool + IncludeMinors bool + IncludeMajors bool + AfterV2 bool + ListMigrations bool + MigrationDirectory string } func (o Options) Filter(tags []string) []string { @@ -108,11 +127,17 @@ func (o Options) Filter(tags []string) []string { } func run(opts Options) error { - tags, err := gitTags() - if err != nil { - return xerrors.Errorf("gitTags: %w", err) + var tags []string + if len(opts.VersionList) > 0 { + tags = opts.VersionList + } else { + var err error + tags, err = gitTags() + if err != nil { + return xerrors.Errorf("gitTags: %w", err) + } + tags = opts.Filter(tags) } - tags = opts.Filter(tags) patches := make([]string, 0) minors := make([]string, 0) @@ -125,7 +150,7 @@ func run(opts Options) error { a := tags[i] b := tags[i+1] - migrations, err := hasMigrationDiff(a, b) + migrations, err := hasMigrationDiff(opts.MigrationDirectory, a, b) if err != nil { return xerrors.Errorf("hasMigrationDiff %q->%q: %w", a, b, err) } @@ -159,7 +184,7 @@ func run(opts Options) error { } if migrations != nil { - log.Printf("[%s] %d migrations added between %s and %s\n", vDiffType, len(migrations)/2, a, b) + log.Printf("[%s] %d migrations added between %s and %s\n", vDiffType, len(migrations), a, b) if opts.ListMigrations { for _, migration := range migrations { log.Printf("\t%s", migration) @@ -188,12 +213,12 @@ func versionDiff(a, b string) string { return "patch" } -func hasMigrationDiff(a, b string) ([]string, error) { +func hasMigrationDiff(dir string, a, b string) ([]string, error) { cmd := exec.Command("git", "diff", // Only added files "--diff-filter=A", "--name-only", - a, b, "coderd/database/migrations") + a, b, dir) output, err := cmd.Output() if err != nil { return nil, xerrors.Errorf("%s\n%s", strings.Join(cmd.Args, " "), err) @@ -203,7 +228,19 @@ func hasMigrationDiff(a, b string) ([]string, error) { } migrations := strings.Split(strings.TrimSpace(string(output)), "\n") - return migrations, nil + filtered := make([]string, 0, len(migrations)) + for _, migration := range migrations { + migration := migration + if strings.Contains(migration, "fixtures") { + continue + } + // Only show the ups + if strings.HasSuffix(migration, ".down.sql") { + continue + } + filtered = append(filtered, migration) + } + return filtered, nil } func gitTags() ([]string, error) { From c07f2fd2455cab099933f5b71cb9352a8e9e8bfc Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Tue, 21 Nov 2023 12:16:46 -0600 Subject: [PATCH 5/6] Linting --- scripts/releasemigrations/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/releasemigrations/main.go b/scripts/releasemigrations/main.go index 545618db1f545..5189af2ce7fbc 100644 --- a/scripts/releasemigrations/main.go +++ b/scripts/releasemigrations/main.go @@ -67,9 +67,9 @@ func main() { } func usage() { - fmt.Println("Usage: releasemigrations [--patches] [--minors] [--majors]") - fmt.Println("Choose at lease one of --patches, --minors, or --majors. You can choose all!") - fmt.Println("Must be run from the coder repo at the root.") + _, _ = fmt.Println("Usage: releasemigrations [--patches] [--minors] [--majors]") + _, _ = fmt.Println("Choose at lease one of --patches, --minors, or --majors. You can choose all!") + _, _ = fmt.Println("Must be run from the coder repo at the root.") } type Options struct { From afa2d948f27940153145f3a086440b2ed4faae32 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Mon, 27 Nov 2023 09:58:26 -0600 Subject: [PATCH 6/6] Add readme to script --- scripts/releasemigrations/README.md | 86 +++++++++++++++++++++++++++++ scripts/releasemigrations/main.go | 2 +- 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 scripts/releasemigrations/README.md diff --git a/scripts/releasemigrations/README.md b/scripts/releasemigrations/README.md new file mode 100644 index 0000000000000..d16e99ca19bc6 --- /dev/null +++ b/scripts/releasemigrations/README.md @@ -0,0 +1,86 @@ +# Migration Releases + +The `main.go` is a program that lists all releases and which migrations are contained with each upgrade. + +# Usage + +```bash +releasemigrations [--patches] [--minors] [--majors] + -after-v2 + Only include releases after v2.0.0 + -dir string + Migration directory (default "coderd/database/migrations") + -list + List migrations + -majors + Include major releases + -minors + Include minor releases + -patches + Include patches releases + -versions string + Comma separated list of versions to use. This skips uses git tag to find tags. +``` + +# Examples + +## Find all migrations between 2 versions + +Going from 2.3.0 to 2.4.0 + +```bash +$ go run scripts/releasemigrations/main.go --list --versions=v2.3.0,v2.4.0 11:47:00 AM +2023/11/21 11:47:09 [minor] 4 migrations added between v2.3.0 and v2.4.0 +2023/11/21 11:47:09 coderd/database/migrations/000165_prevent_autostart_days.up.sql +2023/11/21 11:47:09 coderd/database/migrations/000166_template_active_version.up.sql +2023/11/21 11:47:09 coderd/database/migrations/000167_workspace_agent_api_version.up.sql +2023/11/21 11:47:09 coderd/database/migrations/000168_pg_coord_tailnet_v2_api.up.sql +2023/11/21 11:47:09 Patches: 0 (0 with migrations) +2023/11/21 11:47:09 Minors: 1 (1 with migrations) +2023/11/21 11:47:09 Majors: 0 (0 with migrations) +``` + +## Looking at all patch releases after v2 + +```bash +$ go run scripts/releasemigrations/main.go --patches --after-v2 11:47:09 AM +2023/11/21 11:48:00 [patch] No migrations added between v2.0.0 and v2.0.1 +2023/11/21 11:48:00 [patch] 2 migrations added between v2.0.1 and v2.0.2 +2023/11/21 11:48:00 [patch] No migrations added between v2.1.0 and v2.1.1 +2023/11/21 11:48:00 [patch] No migrations added between v2.1.1 and v2.1.2 +2023/11/21 11:48:00 [patch] No migrations added between v2.1.2 and v2.1.3 +2023/11/21 11:48:00 [patch] 1 migrations added between v2.1.3 and v2.1.4 +2023/11/21 11:48:00 [patch] 2 migrations added between v2.1.4 and v2.1.5 +2023/11/21 11:48:00 [patch] 1 migrations added between v2.3.0 and v2.3.1 +2023/11/21 11:48:00 [patch] 1 migrations added between v2.3.1 and v2.3.2 +2023/11/21 11:48:00 [patch] 1 migrations added between v2.3.2 and v2.3.3 +2023/11/21 11:48:00 Patches: 10 (6 with migrations) +2023/11/21 11:48:00 Minors: 4 (4 with migrations) +2023/11/21 11:48:00 Majors: 0 (0 with migrations) +``` + +## Seeing all the noise this thing can make + +This shows when every migration was introduced. + +```bash +$ go run scripts/releasemigrations/main.go --patches --minors --majors --list +# ... +2023/11/21 11:48:31 [minor] 5 migrations added between v2.2.1 and v2.3.0 +2023/11/21 11:48:31 coderd/database/migrations/000160_provisioner_job_status.up.sql +2023/11/21 11:48:31 coderd/database/migrations/000161_workspace_agent_stats_template_id_created_at_user_id_include_sessions.up.sql +2023/11/21 11:48:31 coderd/database/migrations/000162_workspace_automatic_updates.up.sql +2023/11/21 11:48:31 coderd/database/migrations/000163_external_auth_extra.up.sql +2023/11/21 11:48:31 coderd/database/migrations/000164_archive_template_versions.up.sql +2023/11/21 11:48:31 [patch] 1 migrations added between v2.3.0 and v2.3.1 +2023/11/21 11:48:31 coderd/database/migrations/000165_prevent_autostart_days.up.sql +2023/11/21 11:48:31 [patch] 1 migrations added between v2.3.1 and v2.3.2 +2023/11/21 11:48:31 coderd/database/migrations/000166_template_active_version.up.sql +2023/11/21 11:48:31 [patch] 1 migrations added between v2.3.2 and v2.3.3 +2023/11/21 11:48:31 coderd/database/migrations/000167_workspace_agent_api_version.up.sql +2023/11/21 11:48:31 [minor] 1 migrations added between v2.3.3 and v2.4.0 +2023/11/21 11:48:31 coderd/database/migrations/000168_pg_coord_tailnet_v2_api.up.sql +2023/11/21 11:48:31 Patches: 122 (55 with migrations) +2023/11/21 11:48:31 Minors: 31 (26 with migrations) +2023/11/21 11:48:31 Majors: 1 (1 with migrations) +``` diff --git a/scripts/releasemigrations/main.go b/scripts/releasemigrations/main.go index 5189af2ce7fbc..a06be904b004d 100644 --- a/scripts/releasemigrations/main.go +++ b/scripts/releasemigrations/main.go @@ -67,7 +67,7 @@ func main() { } func usage() { - _, _ = fmt.Println("Usage: releasemigrations [--patches] [--minors] [--majors]") + _, _ = fmt.Println("Usage: releasemigrations [--patches] [--minors] [--majors] [--list]") _, _ = fmt.Println("Choose at lease one of --patches, --minors, or --majors. You can choose all!") _, _ = fmt.Println("Must be run from the coder repo at the root.") }