diff --git a/coderd/util/slice/slice.go b/coderd/util/slice/slice.go index 2cc6c2a4bb0d5..eeda87591e180 100644 --- a/coderd/util/slice/slice.go +++ b/coderd/util/slice/slice.go @@ -22,6 +22,22 @@ func Overlap[T comparable](a []T, b []T) bool { }) } +// Unique returns a new slice with all duplicate elements removed. +// This is a slow function on large lists. +// TODO: Sort elements and implement a faster search algorithm if we +// +// really start to use this. +func Unique[T comparable](a []T) []T { + cpy := make([]T, 0, len(a)) + for _, v := range a { + v := v + if !Contains(cpy, v) { + cpy = append(cpy, v) + } + } + return cpy +} + func OverlapCompare[T any](a []T, b []T, equal func(a, b T) bool) bool { // For each element in b, if at least 1 is contained in 'a', // return true. diff --git a/coderd/util/slice/slice_test.go b/coderd/util/slice/slice_test.go index d69b6c9c440ed..103b9603a272a 100644 --- a/coderd/util/slice/slice_test.go +++ b/coderd/util/slice/slice_test.go @@ -9,6 +9,22 @@ import ( "github.com/coder/coder/coderd/util/slice" ) +func TestUnique(t *testing.T) { + t.Parallel() + + require.ElementsMatch(t, + []int{1, 2, 3, 4, 5}, + slice.Unique([]int{ + 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, + })) + + require.ElementsMatch(t, + []string{"a"}, + slice.Unique([]string{ + "a", "a", "a", + })) +} + func TestContains(t *testing.T) { t.Parallel() diff --git a/scripts/apitypings/main.go b/scripts/apitypings/main.go index dde2057d91895..95bff2b14bda4 100644 --- a/scripts/apitypings/main.go +++ b/scripts/apitypings/main.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "fmt" "go/types" @@ -10,6 +11,9 @@ import ( "regexp" "sort" "strings" + "text/template" + + "github.com/coder/coder/coderd/util/slice" "github.com/fatih/structtag" "golang.org/x/tools/go/packages" @@ -27,20 +31,33 @@ const ( func main() { ctx := context.Background() log := slog.Make(sloghuman.Sink(os.Stderr)) - codeBlocks, err := GenerateFromDirectory(ctx, log, baseDir) + output, err := Generate(baseDir) if err != nil { log.Fatal(ctx, err.Error()) } // Just cat the output to a file to capture it - _, _ = fmt.Println(codeBlocks.String()) + fmt.Println(output) +} + +func Generate(directory string) (string, error) { + ctx := context.Background() + log := slog.Make(sloghuman.Sink(os.Stderr)) + codeBlocks, err := GenerateFromDirectory(ctx, log, directory) + if err != nil { + return "", err + } + + // Just cat the output to a file to capture it + return codeBlocks.String(), nil } // TypescriptTypes holds all the code blocks created. type TypescriptTypes struct { // Each entry is the type name, and it's typescript code block. - Types map[string]string - Enums map[string]string + Types map[string]string + Enums map[string]string + Generics map[string]string } // String just combines all the codeblocks. @@ -50,6 +67,7 @@ func (t TypescriptTypes) String() string { sortedTypes := make([]string, 0, len(t.Types)) sortedEnums := make([]string, 0, len(t.Enums)) + sortedGenerics := make([]string, 0, len(t.Generics)) for k := range t.Types { sortedTypes = append(sortedTypes, k) @@ -57,9 +75,13 @@ func (t TypescriptTypes) String() string { for k := range t.Enums { sortedEnums = append(sortedEnums, k) } + for k := range t.Generics { + sortedGenerics = append(sortedGenerics, k) + } sort.Strings(sortedTypes) sort.Strings(sortedEnums) + sort.Strings(sortedGenerics) for _, k := range sortedTypes { v := t.Types[k] @@ -73,13 +95,20 @@ func (t TypescriptTypes) String() string { _, _ = s.WriteRune('\n') } + for _, k := range sortedGenerics { + v := t.Generics[k] + _, _ = s.WriteString(v) + _, _ = s.WriteRune('\n') + } + return strings.TrimRight(s.String(), "\n") } // GenerateFromDirectory will return all the typescript code blocks for a directory func GenerateFromDirectory(ctx context.Context, log slog.Logger, directory string) (*TypescriptTypes, error) { g := Generator{ - log: log, + log: log, + builtins: make(map[string]string), } err := g.parsePackage(ctx, directory) if err != nil { @@ -98,6 +127,15 @@ type Generator struct { // Package we are scanning. pkg *packages.Package log slog.Logger + + // builtins is kinda a hack to get around the fact that using builtin + // generic constraints is common. We want to support them even though + // they are external to our package. + // It is also a string because the builtins are not proper go types. Meaning + // if you inspect the types, they are not "correct". Things like "comparable" + // cannot be implemented in go. So they are a first class thing that we just + // have to make a static string for ¯\_(ツ)_/¯ + builtins map[string]string } // parsePackage takes a list of patterns such as a directory, and parses them. @@ -128,12 +166,15 @@ func (g *Generator) parsePackage(ctx context.Context, patterns ...string) error // generateAll will generate for all types found in the pkg func (g *Generator) generateAll() (*TypescriptTypes, error) { - structs := make(map[string]string) - enums := make(map[string]types.Object) - enumConsts := make(map[string][]*types.Const) + m := &Maps{ + Structs: make(map[string]string), + Generics: make(map[string]string), + Enums: make(map[string]types.Object), + EnumConsts: make(map[string][]*types.Const), + IgnoredTypes: make(map[string]struct{}), + } // Look for comments that indicate to ignore a type for typescript generation. - ignoredTypes := make(map[string]struct{}) ignoreRegex := regexp.MustCompile("@typescript-ignore[:]?(?P.*)") for _, file := range g.pkg.Syntax { for _, comment := range file.Comments { @@ -144,7 +185,7 @@ func (g *Generator) generateAll() (*TypescriptTypes, error) { if len(matches) >= ignored && matches[ignored] != "" { arr := strings.Split(matches[ignored], ",") for _, s := range arr { - ignoredTypes[strings.TrimSpace(s)] = struct{}{} + m.IgnoredTypes[strings.TrimSpace(s)] = struct{}{} } } } @@ -153,80 +194,24 @@ func (g *Generator) generateAll() (*TypescriptTypes, error) { for _, n := range g.pkg.Types.Scope().Names() { obj := g.pkg.Types.Scope().Lookup(n) - if obj == nil || obj.Type() == nil { - // This would be weird, but it is if the package does not have the type def. - continue - } - - // Exclude ignored types - if _, ok := ignoredTypes[obj.Name()]; ok { - continue + err := g.generateOne(m, obj) + if err != nil { + return nil, xerrors.Errorf("%q: %w", n, err) } + } - switch obj := obj.(type) { - // All named types are type declarations - case *types.TypeName: - named, ok := obj.Type().(*types.Named) - if !ok { - panic("all typename should be named types") - } - switch named.Underlying().(type) { - case *types.Struct: - // type struct - // Structs are obvious. - st, _ := obj.Type().Underlying().(*types.Struct) - codeBlock, err := g.buildStruct(obj, st) - if err != nil { - return nil, xerrors.Errorf("generate %q: %w", obj.Name(), err) - } - structs[obj.Name()] = codeBlock - case *types.Basic: - // type string - // These are enums. Store to expand later. - enums[obj.Name()] = obj - case *types.Map: - // Declared maps that are not structs are still valid codersdk objects. - // Handle them custom by calling 'typescriptType' directly instead of - // iterating through each struct field. - // These types support no json/typescript tags. - // These are **NOT** enums, as a map in Go would never be used for an enum. - ts, err := g.typescriptType(obj.Type().Underlying()) - if err != nil { - return nil, xerrors.Errorf("(map) generate %q: %w", obj.Name(), err) - } - - var str strings.Builder - _, _ = str.WriteString(g.posLine(obj)) - if ts.AboveTypeLine != "" { - str.WriteString(ts.AboveTypeLine) - str.WriteRune('\n') - } - // Use similar output syntax to enums. - str.WriteString(fmt.Sprintf("export type %s = %s\n", obj.Name(), ts.ValueType)) - structs[obj.Name()] = str.String() - case *types.Array, *types.Slice: - // TODO: @emyrk if you need this, follow the same design as "*types.Map" case. - } - case *types.Var: - // TODO: Are any enums var declarations? This is also codersdk.Me. - case *types.Const: - // We only care about named constant types, since they are enums - if named, ok := obj.Type().(*types.Named); ok { - name := named.Obj().Name() - enumConsts[name] = append(enumConsts[name], obj) - } - case *types.Func: - // Noop - default: - fmt.Println(obj.Name()) + // Add the builtins + for n, value := range g.builtins { + if value != "" { + m.Generics[n] = value } } // Write all enums enumCodeBlocks := make(map[string]string) - for name, v := range enums { + for name, v := range m.Enums { var values []string - for _, elem := range enumConsts[name] { + for _, elem := range m.EnumConsts[name] { // TODO: If we have non string constants, we need to handle that // here. values = append(values, elem.Val().String()) @@ -242,21 +227,181 @@ func (g *Generator) generateAll() (*TypescriptTypes, error) { } return &TypescriptTypes{ - Types: structs, - Enums: enumCodeBlocks, + Types: m.Structs, + Enums: enumCodeBlocks, + Generics: m.Generics, }, nil } +type Maps struct { + Structs map[string]string + Generics map[string]string + Enums map[string]types.Object + EnumConsts map[string][]*types.Const + IgnoredTypes map[string]struct{} +} + +func (g *Generator) generateOne(m *Maps, obj types.Object) error { + if obj == nil || obj.Type() == nil { + // This would be weird, but it is if the package does not have the type def. + return nil + } + + // Exclude ignored types + if _, ok := m.IgnoredTypes[obj.Name()]; ok { + return nil + } + + switch obj := obj.(type) { + // All named types are type declarations + case *types.TypeName: + named, ok := obj.Type().(*types.Named) + if !ok { + panic("all typename should be named types") + } + switch underNamed := named.Underlying().(type) { + case *types.Struct: + // type struct + // Structs are obvious. + codeBlock, err := g.buildStruct(obj, underNamed) + if err != nil { + return xerrors.Errorf("generate %q: %w", obj.Name(), err) + } + m.Structs[obj.Name()] = codeBlock + case *types.Basic: + // type string + // These are enums. Store to expand later. + m.Enums[obj.Name()] = obj + case *types.Map: + // Declared maps that are not structs are still valid codersdk objects. + // Handle them custom by calling 'typescriptType' directly instead of + // iterating through each struct field. + // These types support no json/typescript tags. + // These are **NOT** enums, as a map in Go would never be used for an enum. + ts, err := g.typescriptType(obj.Type().Underlying()) + if err != nil { + return xerrors.Errorf("(map) generate %q: %w", obj.Name(), err) + } + + var str strings.Builder + _, _ = str.WriteString(g.posLine(obj)) + if ts.AboveTypeLine != "" { + str.WriteString(ts.AboveTypeLine) + str.WriteRune('\n') + } + // Use similar output syntax to enums. + str.WriteString(fmt.Sprintf("export type %s = %s\n", obj.Name(), ts.ValueType)) + m.Structs[obj.Name()] = str.String() + case *types.Array, *types.Slice: + // TODO: @emyrk if you need this, follow the same design as "*types.Map" case. + case *types.Interface: + // Interfaces are used as generics. Non-generic interfaces are + // not supported. + if underNamed.NumEmbeddeds() == 1 { + union, ok := underNamed.EmbeddedType(0).(*types.Union) + if !ok { + // If the underlying is not a union, but has 1 type. It's + // just that one type. + union = types.NewUnion([]*types.Term{ + // Set the tilde to true to support underlying. + // Doesn't actually affect our generation. + types.NewTerm(true, underNamed.EmbeddedType(0)), + }) + } + + block, err := g.buildUnion(obj, union) + if err != nil { + return xerrors.Errorf("generate union %q: %w", obj.Name(), err) + } + m.Generics[obj.Name()] = block + } + case *types.Signature: + // Ignore named functions. + default: + // If you hit this error, you added a new unsupported named type. + // The easiest way to solve this is add a new case above with + // your type and a TODO to implement it. + return xerrors.Errorf("unsupported named type %q", underNamed.String()) + } + case *types.Var: + // TODO: Are any enums var declarations? This is also codersdk.Me. + case *types.Const: + // We only care about named constant types, since they are enums + if named, ok := obj.Type().(*types.Named); ok { + name := named.Obj().Name() + m.EnumConsts[name] = append(m.EnumConsts[name], obj) + } + case *types.Func: + // Noop + default: + fmt.Println(obj.Name()) + } + return nil +} + func (g *Generator) posLine(obj types.Object) string { file := g.pkg.Fset.File(obj.Pos()) return fmt.Sprintf("// From %s\n", filepath.Join("codersdk", filepath.Base(file.Name()))) } // buildStruct just prints the typescript def for a type. -func (g *Generator) buildStruct(obj types.Object, st *types.Struct) (string, error) { +func (g *Generator) buildUnion(obj types.Object, st *types.Union) (string, error) { var s strings.Builder _, _ = s.WriteString(g.posLine(obj)) - _, _ = s.WriteString(fmt.Sprintf("export interface %s ", obj.Name())) + + allTypes := make([]string, 0, st.Len()) + var optional bool + for i := 0; i < st.Len(); i++ { + term := st.Term(i) + scriptType, err := g.typescriptType(term.Type()) + if err != nil { + return "", xerrors.Errorf("union %q for %q failed to get type: %w", st.String(), obj.Name(), err) + } + allTypes = append(allTypes, scriptType.ValueType) + optional = optional || scriptType.Optional + } + + if optional { + allTypes = append(allTypes, "null") + } + + allTypes = slice.Unique(allTypes) + + s.WriteString(fmt.Sprintf("export type %s = %s\n", obj.Name(), strings.Join(allTypes, " | "))) + + return s.String(), nil +} + +type structTemplateState struct { + PosLine string + Name string + Fields []string + Generics []string + Extends string + AboveLine string +} + +const structTemplate = `{{ .PosLine -}} +{{ if .AboveLine }}{{ .AboveLine }} +{{ end }}export interface {{ .Name }}{{ if .Generics }}<{{ join .Generics ", " }}>{{ end }}{{ if .Extends }} extends {{ .Extends }}{{ end }} { +{{ join .Fields "\n"}} +} +` + +// buildStruct just prints the typescript def for a type. +func (g *Generator) buildStruct(obj types.Object, st *types.Struct) (string, error) { + state := structTemplateState{} + tpl := template.New("struct") + tpl.Funcs(template.FuncMap{ + "join": strings.Join, + }) + tpl, err := tpl.Parse(structTemplate) + if err != nil { + return "", xerrors.Errorf("parse struct template: %w", err) + } + + state.PosLine = g.posLine(obj) + state.Name = obj.Name() // Handle named embedded structs in the codersdk package via extension. var extends []string @@ -272,10 +417,10 @@ func (g *Generator) buildStruct(obj types.Object, st *types.Struct) (string, err } } if len(extends) > 0 { - _, _ = s.WriteString(fmt.Sprintf("extends %s ", strings.Join(extends, ", "))) + state.Extends = strings.Join(extends, ", ") } - _, _ = s.WriteString("{\n") + genericsUsed := make(map[string]string) // For each field in the struct, we print 1 line of the typescript interface for i := 0; i < st.NumFields(); i++ { if extendedFields[i] { @@ -335,21 +480,48 @@ func (g *Generator) buildStruct(obj types.Object, st *types.Struct) (string, err } if tsType.AboveTypeLine != "" { - _, _ = s.WriteString(tsType.AboveTypeLine) - _, _ = s.WriteRune('\n') + state.AboveLine = tsType.AboveTypeLine } optional := "" if jsonOptional || tsType.Optional { optional = "?" } - _, _ = s.WriteString(fmt.Sprintf("%sreadonly %s%s: %s\n", indent, jsonName, optional, tsType.ValueType)) + valueType := tsType.ValueType + if tsType.GenericValue != "" { + valueType = tsType.GenericValue + for name, constraint := range tsType.GenericTypes { + if _, ok := genericsUsed[name]; ok { + // Don't add a generic twice + // TODO: We should probably check that the generic mapping is + // not a different type. Like 'T' being referenced to 2 different + // constraints. I don't think this is possible though in valid + // go, so I'm going to ignore this for now. + continue + } + state.Generics = append(state.Generics, fmt.Sprintf("%s extends %s", name, constraint)) + genericsUsed[name] = constraint + } + } + state.Fields = append(state.Fields, fmt.Sprintf("%sreadonly %s%s: %s", indent, jsonName, optional, valueType)) } - _, _ = s.WriteString("}\n") - return s.String(), nil + + data := bytes.NewBuffer(make([]byte, 0)) + err = tpl.Execute(data, state) + if err != nil { + return "", xerrors.Errorf("execute struct template: %w", err) + } + return data.String(), nil } type TypescriptType struct { - ValueType string + // GenericValue gives a unique character for mapping the value type + // to a generic. This is only useful if you can use generic syntax. + // This is optional, as the ValueType will have the correct constraints. + GenericValue string + // GenericTypes is a map of generic name to actual constraint + // Example: 'C = comparable'. + GenericTypes map[string]string + ValueType string // AboveTypeLine lets you put whatever text you want above the typescript // type line. AboveTypeLine string @@ -460,10 +632,40 @@ func (g *Generator) typescriptType(ty types.Type) (TypescriptType, error) { // put the name as it will be defined in the typescript codeblock // we generate. name := n.Obj().Name() + genericName := "" + genericTypes := make(map[string]string) if obj := g.pkg.Types.Scope().Lookup(name); obj != nil { // Sweet! Using other typescript types as fields. This could be an // enum or another struct - return TypescriptType{ValueType: name}, nil + if args := n.TypeArgs(); args != nil && args.Len() > 0 { + genericConstraints := make([]string, 0, args.Len()) + genericNames := make([]string, 0, args.Len()) + for i := 0; i < args.Len(); i++ { + genType, err := g.typescriptType(args.At(i)) + if err != nil { + return TypescriptType{}, xerrors.Errorf("generic field %q<%q>: %w", name, args.At(i).String(), err) + } + + if param, ok := args.At(i).(*types.TypeParam); ok { + // Using a generic defined by the parent. + gname := param.Obj().Name() + genericNames = append(genericNames, gname) + genericTypes[gname] = genType.ValueType + } else { + // Defining a generic + genericNames = append(genericNames, genType.ValueType) + } + + genericConstraints = append(genericConstraints, genType.ValueType) + } + genericName = name + fmt.Sprintf("<%s>", strings.Join(genericNames, ", ")) + name += fmt.Sprintf("<%s>", strings.Join(genericConstraints, ", ")) + } + return TypescriptType{ + GenericTypes: genericTypes, + GenericValue: genericName, + ValueType: name, + }, nil } // If it's a struct, just use the name of the struct type @@ -498,6 +700,50 @@ func (g *Generator) typescriptType(ty types.Type) (TypescriptType, error) { AboveTypeLine: indentedComment("eslint-disable-next-line")}, nil } return TypescriptType{}, xerrors.New("only empty interface types are supported") + case *types.TypeParam: + _, ok := ty.Underlying().(*types.Interface) + if !ok { + // If it's not an interface, it is likely a usage of generics that + // we have not hit yet. Feel free to add support for it. + return TypescriptType{}, xerrors.New("type param must be an interface") + } + + generic := ty.Constraint() + // We don't mess with multiple packages, so just trim the package path + // from the name. + pkgPath := ty.Obj().Pkg().Path() + name := strings.TrimPrefix(generic.String(), pkgPath+".") + + referenced := g.pkg.Types.Scope().Lookup(name) + + if referenced == nil { + include, builtinString := g.isBuiltIn(name) + if !include { + // If we don't have the type constraint defined somewhere in the package, + // then we have to resort to using any. + return TypescriptType{ + GenericTypes: map[string]string{ + ty.Obj().Name(): "any", + }, + GenericValue: ty.Obj().Name(), + ValueType: "any", + AboveTypeLine: fmt.Sprintf("// %q is an external type, so we use any", name), + Optional: false, + }, nil + } + // Include the builtin for this type to reference + g.builtins[name] = builtinString + } + + return TypescriptType{ + GenericTypes: map[string]string{ + ty.Obj().Name(): name, + }, + GenericValue: ty.Obj().Name(), + ValueType: name, + AboveTypeLine: "", + Optional: false, + }, nil } // These are all the other types we need to support. @@ -505,6 +751,24 @@ func (g *Generator) typescriptType(ty types.Type) (TypescriptType, error) { return TypescriptType{}, xerrors.Errorf("unknown type: %s", ty.String()) } +// isBuiltIn returns the string for a builtin type that we want to support +// if the name is a reserved builtin type. This is for types like 'comparable'. +// These types are not implemented in golang, so we just have to hardcode it. +func (g *Generator) isBuiltIn(name string) (bool, string) { + // Note: @emyrk If we use constraints like Ordered, we can pull those + // dynamically from their respective packages. + switch name { + case "comparable": + // To be complete, we include "any". Kinda sucks :( + return true, "export type comparable = boolean | number | string | any" + case "any": + // This is supported in typescript, we don't need to write anything + return true, "" + default: + return false, "" + } +} + func indentedComment(comment string) string { return fmt.Sprintf("%s// %s", indent, comment) } diff --git a/scripts/apitypings/main_test.go b/scripts/apitypings/main_test.go new file mode 100644 index 0000000000000..c7fabede3c4e8 --- /dev/null +++ b/scripts/apitypings/main_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGeneration(t *testing.T) { + files, err := os.ReadDir("testdata") + require.NoError(t, err, "read dir") + + for _, f := range files { + if !f.IsDir() { + // Only test directories + continue + } + f := f + t.Run(f.Name(), func(t *testing.T) { + dir := filepath.Join(".", "testdata", f.Name()) + output, err := Generate("./" + dir) + require.NoErrorf(t, err, "generate %q", dir) + + golden := filepath.Join(dir, f.Name()+".ts") + expected, err := os.ReadFile(golden) + require.NoErrorf(t, err, "read file %s", golden) + expectedString := strings.TrimSpace(string(expected)) + output = strings.TrimSpace(output) + require.Equal(t, expectedString, output, "matched output") + }) + } +} diff --git a/scripts/apitypings/testdata/generics/generics.go b/scripts/apitypings/testdata/generics/generics.go new file mode 100644 index 0000000000000..0bed9ed059928 --- /dev/null +++ b/scripts/apitypings/testdata/generics/generics.go @@ -0,0 +1,30 @@ +package generics + +import "time" + +type Single interface { + string +} + +type Custom interface { + string | bool | int | time.Duration | []string | *int +} + +// StaticGeneric has all generic fields defined in the field +type StaticGeneric struct { + Static GenericFields[string, int, time.Duration] `json:"static"` +} + +// DynamicGeneric can has some dynamic fields +type DynamicGeneric[C comparable, A any] struct { + Dynamic GenericFields[C, A, string] `json:"dynamic"` + Comparable C `json:"comparable"` +} + +type GenericFields[C comparable, A any, T Custom] struct { + Comparable C `json:"comparable"` + Any A `json:"any"` + + Custom T `json:"custom"` + Again T `json:"again"` +} diff --git a/scripts/apitypings/testdata/generics/generics.ts b/scripts/apitypings/testdata/generics/generics.ts new file mode 100644 index 0000000000000..8896afb764c62 --- /dev/null +++ b/scripts/apitypings/testdata/generics/generics.ts @@ -0,0 +1,28 @@ +// Code generated by 'make coder/scripts/apitypings/main.go'. DO NOT EDIT. + +// From codersdk/generics.go +export interface DynamicGeneric { + readonly dynamic: GenericFields + readonly comparable: C +} + +// From codersdk/generics.go +export interface GenericFields { + readonly comparable: C + readonly any: A + readonly custom: T + readonly again: T +} + +// From codersdk/generics.go +export interface StaticGeneric { + readonly static: GenericFields +} + +// From codersdk/generics.go +export type Custom = string | boolean | number | string[] | null + +// From codersdk/generics.go +export type Single = string + +export type comparable = boolean | number | string | any