Skip to content

Update the auto YAML Generation #7725

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 34 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
1df036d
some changes for connector YAML
royendo Aug 4, 2025
3ff888e
remove
royendo Aug 4, 2025
a381593
separated YAML schema
royendo Aug 4, 2025
b6b987f
nit
royendo Aug 4, 2025
d7d4b11
link fixes
royendo Aug 4, 2025
a3784a0
Update generate_project.go
royendo Aug 4, 2025
f2ed2c4
add sec warning
royendo Aug 4, 2025
d57d430
motherduck live connector
royendo Aug 4, 2025
e5434ee
remove local file from connector
royendo Aug 4, 2025
cd0c819
nit
royendo Aug 4, 2025
d3396eb
matching file names
royendo Aug 4, 2025
16c99c6
reordered files and added sources YAML and model SQL
royendo Aug 4, 2025
6ba0dcd
nit, fix links
royendo Aug 4, 2025
4363fff
comparing with current YAML changes
royendo Aug 5, 2025
64677e0
Merge branch 'main' into docs/auto-gen-YAML
royendo Aug 5, 2025
77f0ca9
adding annotatons back
royendo Aug 5, 2025
7c35559
gofmt, golint
royendo Aug 5, 2025
0ff11e2
fix
royendo Aug 5, 2025
11b2710
first pass fix, need to review a few other items
royendo Aug 6, 2025
db75a6b
remove toplevel driver
royendo Aug 6, 2025
95ca49a
single project file
royendo Aug 6, 2025
98baf56
Update generate_project.go
royendo Aug 6, 2025
7065bcc
good old cursor wiped my functoni
royendo Aug 6, 2025
dba395a
missing parameters
royendo Aug 6, 2025
6e32ddd
midding parameters and inline examples
royendo Aug 6, 2025
c55eff3
god so many false positives from asking cursor to merge files
royendo Aug 6, 2025
6ffe7f8
adding links
royendo Aug 6, 2025
f69c89a
fixed oneOf
royendo Aug 6, 2025
3b6517e
split data properties to have api excamples
royendo Aug 6, 2025
e67f159
gonig through files
royendo Aug 7, 2025
df54d71
returning original format
royendo Aug 7, 2025
f94396e
?
royendo Aug 7, 2025
c58c634
Merge remote-tracking branch 'origin/main' into docs/auto-gen-YAML
royendo Aug 8, 2025
e8c04e9
fixing urls,
royendo Aug 8, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
good old cursor wiped my functoni
  • Loading branch information
royendo committed Aug 6, 2025
commit 7065bcc181a62931f77671f621d0bc4b0eb02dba
88 changes: 88 additions & 0 deletions cli/cmd/docs/generate_project.go
Original file line number Diff line number Diff line change
Expand Up @@ -446,8 +446,12 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req
for _, item := range oneOf.Content {
// Since $ref values are already resolved, look for connector definitions directly
title := getScalarValue(item, "title")
exampleOutput := generateConnectorExample(title, item)

if title != "" {
doc.WriteString(fmt.Sprintf("\n\n### %s\n\n", title))
// Generate copiable example for this connector
doc.WriteString(fmt.Sprintf("\n\n\n%s", exampleOutput))
// Add description first
description := getPrintableDescription(item, indent, "")
if description != "" {
Expand All @@ -472,6 +476,8 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req
required))
}
}


}

} else {
Expand Down Expand Up @@ -561,6 +567,8 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req
doc.WriteString(fmt.Sprintf("%s\n\n", description))
}
}
exampleOutput := generateConnectorExample(title, connectorDef)
doc.WriteString(fmt.Sprintf("\n\n#### Example\n\n%s", exampleOutput))

// Generate the connector definition documentation (properties, etc.) but skip the header
// We need to process properties manually to avoid duplicate headers
Expand All @@ -580,6 +588,7 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req
required))
}
}

}
}

Expand Down Expand Up @@ -620,3 +629,82 @@ func hasCombinators(node *yaml.Node) bool {
return getNodeForKey(node, "anyOf") != nil || getNodeForKey(node, "oneOf") != nil || getNodeForKey(node, "allOf") != nil
}

func generateConnectorExample(connectorType string, connectorDef *yaml.Node) string {
if connectorDef == nil {
return ""
}

var example strings.Builder
example.WriteString("```yaml\n")
example.WriteString("type: connector # Must be `connector` (required)\n")

// Get the driver from the schema and add it first
driverAdded := false
if driver := getNodeForKey(connectorDef, "driver"); driver != nil {
if constVal := getScalarValue(driver, "const"); constVal != "" {
example.WriteString(fmt.Sprintf("driver: %s # Must be `%s` _(required)_\n\n", constVal, constVal))
driverAdded = true
}
}

// Fallback: if driver wasn't found, use the connector type name
if !driverAdded {
// Special case for MotherDuck which uses duckdb driver
if connectorType == "MotherDuck" {
example.WriteString("driver: duckdb # Must be `duckdb` _(required)_\n\n")
} else {
example.WriteString(fmt.Sprintf("driver: %s # Must be `%s` _(required)_\n\n", strings.ToLower(connectorType), strings.ToLower(connectorType)))
}
}

// Get all properties from the schema
if properties := getNodeForKey(connectorDef, "properties"); properties != nil && properties.Kind == yaml.MappingNode {
for i := 0; i < len(properties.Content); i += 2 {
propName := properties.Content[i].Value
propValue := properties.Content[i+1]

// Skip the driver property since we already added it
if propName == "driver" {
continue
}

// Get property description
description := getPrintableDescription(propValue, "", "")
if description == "" {
description = "Property description"
}

// Get sample value from the schema
sampleValue := getScalarValue(propValue, "sample")
if sampleValue == "" {
// Fallback to const value if no sample
sampleValue = getScalarValue(propValue, "const")
}
if sampleValue == "" {
// Final fallback
sampleValue = "example_value"
}

// Check if it's required
required := ""
if requiredFields := getRequiredMapFromNode(connectorDef); requiredFields[propName] {
required = " _(required)_"
}

// Format the line with proper alignment
example.WriteString(fmt.Sprintf("%s: %s", propName, sampleValue))

// Add padding for alignment
padding := 35 - len(propName) - len(sampleValue)
if padding > 0 {
example.WriteString(strings.Repeat(" ", padding))
}

example.WriteString(fmt.Sprintf("# %s%s\n", description, required))
}
}

example.WriteString("```\n\n")
return example.String()
}

Loading