From 1df036de62e89b25291b95a3a2ffb2459b938350 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 4 Aug 2025 11:01:36 -0600 Subject: [PATCH 01/32] some changes for connector YAML unsur eif schema./yaml is maintained or auto generated. --- cli/cmd/docs/generate_project.go | 284 ++++++++++++++++++++-- runtime/parser/schema/project.schema.yaml | 14 +- test-output/theme.md | 52 ++++ 3 files changed, 327 insertions(+), 23 deletions(-) create mode 100644 test-output/theme.md diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index fcc5668e687..84a007457ed 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -320,6 +320,10 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req currentLevel := level title := getScalarValue(node, "title") description := getPrintableDescription(node, indent, "") + + // Check if this is a connector type for special handling + isConnector := title == "Connector YAML" + if level == 0 { doc.WriteString("---\n") doc.WriteString("note: GENERATED. DO NOT EDIT.\n") @@ -329,6 +333,8 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req if description != "" { doc.WriteString(fmt.Sprintf("\n\n%s", description)) } + + level++ // level zero is to print base level info and its only onetime for a page so increasing level } else if level == 1 { if title != "" { @@ -378,25 +384,33 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req // OneOf if oneOf := getNodeForKey(node, "oneOf"); oneOf != nil && oneOf.Kind == yaml.SequenceNode { - if len(oneOf.Content) == 1 { - doc.WriteString(generateDoc(sidebarPosition, level, oneOf.Content[0], indent, getRequiredMapFromNode(oneOf.Content[0]))) - } else { - if level == 1 { - doc.WriteString("\n\n## One of Properties Options") - for _, item := range oneOf.Content { - title := getScalarValue(item, "title") - if title != "" { - anchor := strings.ToLower(strings.ReplaceAll(title, " ", "-")) - doc.WriteString(fmt.Sprintf("\n- [%s](#%s)", title, anchor)) + // Skip oneOf processing for connectors at level 1 since we handle it in allOf + if !(isConnector && level == 1) { + if len(oneOf.Content) == 1 { + doc.WriteString(generateDoc(sidebarPosition, level, oneOf.Content[0], indent, getRequiredMapFromNode(oneOf.Content[0]))) + } else { + if level == 1 { + doc.WriteString("\n\n## One of Properties Options") + for _, item := range oneOf.Content { + title := getScalarValue(item, "title") + if title != "" { + anchor := strings.ToLower(strings.ReplaceAll(title, " ", "-")) + doc.WriteString(fmt.Sprintf("\n- [%s](#%s)", title, anchor)) + } } } - for _, item := range oneOf.Content { - doc.WriteString(generateDoc(sidebarPosition, level, item, indent, getRequiredMapFromNode(item))) - } - } else { + for i, item := range oneOf.Content { if hasType(item) || hasProperties(item) || hasCombinators(item) { - doc.WriteString(fmt.Sprintf("\n\n%s- **option %d** - %s - %s", indent, i+1, getPrintableType(item), getPrintableDescription(item, indent, "(no description)"))) + title := getScalarValue(item, "title") + if title != "" { + doc.WriteString(fmt.Sprintf("\n\n#### Option %d: %s", i+1, title)) + } else { + doc.WriteString(fmt.Sprintf("\n\n#### Option %d", i+1)) + } + doc.WriteString(fmt.Sprintf("\n\n**Type:** %s\n\n**Description:** %s", + getPrintableType(item), + getPrintableDescription(item, indent, "(no description)"))) doc.WriteString(generateDoc(sidebarPosition, level, item, indent+" ", getRequiredMapFromNode(item))) } } @@ -416,7 +430,55 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req // AllOf if allOf := getNodeForKey(node, "allOf"); allOf != nil && allOf.Kind == yaml.SequenceNode { + // Special handling for connector allOf + if isConnector && level == 1 { + doc.WriteString("\n\n## Available Connector Types\n\n") + doc.WriteString("Choose from the following connector types based on your data source:\n\n") + + // Find the oneOf section within allOf + for _, item := range allOf.Content { + if oneOf := getNodeForKey(item, "oneOf"); oneOf != nil && oneOf.Kind == yaml.SequenceNode { + doc.WriteString("### OLAP Engines\n\n") + doc.WriteString("- [**DuckDB**](#duckdb) - Embedded DuckDB engine (default)\n") + doc.WriteString("- [**ClickHouse**](#clickhouse) - ClickHouse analytical database\n") + doc.WriteString("- [**MotherDuck**](#motherduck) - MotherDuck cloud database\n") + doc.WriteString("- [**Druid**](#druid) - Apache Druid\n") + doc.WriteString("- [**Pinot**](#pinot) - Apache Pinot\n\n") + + doc.WriteString("### Data Warehouses\n\n") + doc.WriteString("- [**Snowflake**](#snowflake) - Snowflake data warehouse\n") + doc.WriteString("- [**BigQuery**](#bigquery) - Google BigQuery\n") + doc.WriteString("- [**Redshift**](#redshift) - Amazon Redshift\n") + doc.WriteString("- [**Athena**](#athena) - Amazon Athena\n\n") + + doc.WriteString("### Databases\n\n") + doc.WriteString("- [**PostgreSQL**](#postgres) - PostgreSQL databases\n") + doc.WriteString("- [**MySQL**](#mysql) - MySQL databases\n") + doc.WriteString("- [**SQLite**](#sqlite) - SQLite databases\n\n") + + doc.WriteString("### Cloud Storage\n\n") + doc.WriteString("- [**GCS**](#gcs) - Google Cloud Storage\n") + doc.WriteString("- [**S3**](#s3) - Amazon S3 storage\n") + doc.WriteString("- [**Azure**](#azure) - Azure Blob Storage\n\n") + + doc.WriteString("### Other\n\n") + doc.WriteString("- [**HTTPS**](#https) - Public files via HTTP/HTTPS\n") + doc.WriteString("- [**Salesforce**](#salesforce) - Salesforce data\n") + doc.WriteString("- [**Slack**](#slack) - Slack data\n") + doc.WriteString("- [**Local File**](#local_file) - Local file system\n\n") + + doc.WriteString("## Connector Details\n\n") + break + } + } + } + for _, item := range allOf.Content { + // Skip oneOf items for connectors since we handle them separately + if isConnector && getNodeForKey(item, "oneOf") != nil { + continue + } + if hasIf(item) { ifNode := getNodeForKey(item, "if") title := getScalarValue(ifNode, "title") @@ -433,6 +495,44 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req } } + // Definitions (for connectors) + if definitions := getNodeForKey(node, "definitions"); definitions != nil && definitions.Kind == yaml.MappingNode && isConnector && level == 1 { + for i := 0; i < len(definitions.Content); i += 2 { + connectorDef := definitions.Content[i+1] + + title := getScalarValue(connectorDef, "title") + if title != "" { + doc.WriteString(fmt.Sprintf("\n\n## %s\n\n", title)) + // Add description first + description := getPrintableDescription(connectorDef, indent, "") + if description != "" { + doc.WriteString(fmt.Sprintf("%s\n\n", description)) + } + // Add YAML example after description + doc.WriteString(generateConnectorExample(title, connectorDef)) + } + + // Generate the connector definition documentation (properties, etc.) but skip the header + // We need to process properties manually to avoid duplicate headers + if properties := getNodeForKey(connectorDef, "properties"); properties != nil && properties.Kind == yaml.MappingNode { + for j := 0; j < len(properties.Content); j += 2 { + propName := properties.Content[j].Value + propValue := properties.Content[j+1] + required := "" + if requiredFields := getRequiredMapFromNode(connectorDef); requiredFields[propName] { + required = "_(required)_" + } + + doc.WriteString(fmt.Sprintf("\n\n### `%s`\n\n", propName)) + doc.WriteString(fmt.Sprintf("**Type:** %s\n\n**Description:** %s\n\n%s", + getPrintableType(propValue), + getPrintableDescription(propValue, indent, "(no description)"), + required)) + } + } + } + } + // Examples if examples := getNodeForKey(node, "examples"); examples != nil && examples.Kind == yaml.SequenceNode && currentLevel == 0 { doc.WriteString("\n\n## Examples") @@ -463,3 +563,157 @@ func hasProperties(node *yaml.Node) bool { func hasCombinators(node *yaml.Node) bool { return getNodeForKey(node, "anyOf") != nil || getNodeForKey(node, "oneOf") != nil || getNodeForKey(node, "allOf") != nil } + +func contains(slice []string, item string) bool { + for _, s := range slice { + if s == item { + return true + } + } + return false +} + +func generateConnectorExample(connectorType string, connectorDef *yaml.Node) string { + if connectorDef == nil { + return "" + } + + var example strings.Builder + example.WriteString("```yaml\n") + example.WriteString("type: connector # Must be `connector` (required)\n") + + // Get the driver from the schema and add it first + driverAdded := false + if driver := getNodeForKey(connectorDef, "driver"); driver != nil { + if constVal := getScalarValue(driver, "const"); constVal != "" { + example.WriteString(fmt.Sprintf("driver: %s # Must be `%s` _(required)_\n\n", constVal, constVal)) + driverAdded = true + } + } + + // Fallback: if driver wasn't found, use the connector type name + if !driverAdded { + example.WriteString(fmt.Sprintf("driver: %s # Must be `%s` _(required)_\n\n", strings.ToLower(connectorType), strings.ToLower(connectorType))) + } + + // Get all properties from the schema + if properties := getNodeForKey(connectorDef, "properties"); properties != nil && properties.Kind == yaml.MappingNode { + for i := 0; i < len(properties.Content); i += 2 { + propName := properties.Content[i].Value + propValue := properties.Content[i+1] + + // Skip the driver property since we already added it + if propName == "driver" { + continue + } + + // Get property description + description := getPrintableDescription(propValue, "", "") + if description == "" { + description = "Property description" + } + + // Get property type and generate appropriate example value + propType := getScalarValue(propValue, "type") + exampleValue := generateExampleValue(propName, propType, propValue) + + // Check if it's required + required := "" + if requiredFields := getRequiredMapFromNode(connectorDef); requiredFields[propName] { + required = " _(required)_" + } + + // Format the line with proper alignment + example.WriteString(fmt.Sprintf("%s: %s", propName, exampleValue)) + + // Add padding for alignment + padding := 35 - len(propName) - len(exampleValue) + if padding > 0 { + example.WriteString(strings.Repeat(" ", padding)) + } + + example.WriteString(fmt.Sprintf("# %s%s\n", description, required)) + } + } + + example.WriteString("```\n\n") + return example.String() +} + +func generateExampleValue(propName, propType string, propNode *yaml.Node) string { + // Check for const values first + if constVal := getScalarValue(propNode, "const"); constVal != "" { + return fmt.Sprintf("\"%s\"", constVal) + } + + // Check for enum values + if enum := getNodeForKey(propNode, "enum"); enum != nil && enum.Kind == yaml.SequenceNode && len(enum.Content) > 0 { + return fmt.Sprintf("\"%s\"", enum.Content[0].Value) + } + + // Generate based on type and property name + switch propType { + case "string": + // Generate contextual examples based on property name + switch { + case strings.Contains(propName, "key") && strings.Contains(propName, "secret"): + return "\"myawssecretkey\"" + case strings.Contains(propName, "key") && strings.Contains(propName, "access"): + return "\"myawsaccesskey\"" + case strings.Contains(propName, "token"): + return "\"mytemporarytoken\"" + case strings.Contains(propName, "password"): + return "\"mypassword\"" + case strings.Contains(propName, "username"): + return "\"myusername\"" + case strings.Contains(propName, "host"): + return "\"localhost\"" + case strings.Contains(propName, "port"): + return "5432" + case strings.Contains(propName, "database"): + return "\"mydatabase\"" + case strings.Contains(propName, "bucket"): + return "\"my-bucket\"" + case strings.Contains(propName, "region"): + return "\"us-east-1\"" + case strings.Contains(propName, "workgroup"): + return "\"primary\"" + case strings.Contains(propName, "dsn"): + return "\"postgresql://user:pass@localhost:5432/db\"" + case strings.Contains(propName, "path"): + return "\"/path/to/file\"" + case strings.Contains(propName, "uri"): + return "\"s3://bucket/path\"" + case strings.Contains(propName, "endpoint"): + return "\"https://api.example.com\"" + case strings.Contains(propName, "account"): + return "\"myaccount\"" + case strings.Contains(propName, "project"): + return "\"myproject\"" + case strings.Contains(propName, "cluster"): + return "\"mycluster\"" + case strings.Contains(propName, "role") && strings.Contains(propName, "arn"): + return "\"arn:aws:iam::123456789012:role/MyRole\"" + case strings.Contains(propName, "session"): + return "\"MySession\"" + case strings.Contains(propName, "external"): + return "\"MyExternalID\"" + case strings.Contains(propName, "location"): + return "\"s3://my-bucket/athena-output/\"" + default: + return "\"example_value\"" + } + case "integer": + return "123" + case "boolean": + // Check if there's a default value + if defaultVal := getScalarValue(propNode, "default"); defaultVal != "" { + return defaultVal + } + return "true" + case "number": + return "123.45" + default: + return "\"example_value\"" + } +} \ No newline at end of file diff --git a/runtime/parser/schema/project.schema.yaml b/runtime/parser/schema/project.schema.yaml index ee3a7ce29f9..a34ecd1deaa 100644 --- a/runtime/parser/schema/project.schema.yaml +++ b/runtime/parser/schema/project.schema.yaml @@ -352,13 +352,7 @@ definitions: type: object title: Connector YAML description: | - When you add olap_connector to your rill.yaml file, you will need to set up a `.yaml` file in the 'connectors' directory. This file requires the following parameters,type and driver (see below for more parameter options). Rill will automatically test the connectivity to the OLAP engine upon saving the file. This can be viewed in the connectors tab in the UI. - - :::tip Did you know? - - Starting from Rill 0.46, you can directly create OLAP engines from the UI! Select + Add -> Data -> Connect an OLAP engine - - ::: + Connector YAML files define how Rill connects to external data sources and OLAP engines. Each connector specifies a driver type and its required connection parameters. allOf: - title: Properties type: object @@ -367,9 +361,13 @@ definitions: type: string const: connector description: Refers to the resource type and must be `connector` + driver: + type: string + description: The type of connector, see [available connectors](#available-connector-types) (required) required: - type - - $ref: '#/definitions/common_properties' + - driver + # - $ref: '#/definitions/common_properties' - oneOf: - $ref: '#/definitions/connector/definitions/athena' - $ref: '#/definitions/connector/definitions/azure' diff --git a/test-output/theme.md b/test-output/theme.md new file mode 100644 index 00000000000..35d35542419 --- /dev/null +++ b/test-output/theme.md @@ -0,0 +1,52 @@ +--- +note: GENERATED. DO NOT EDIT. +title: Theme YAML +sidebar_position: 39 +--- + +In your Rill project directory, create a `.yaml` file in any directory containing `type: theme`. Rill will automatically ingest the theme next time you run `rill start` or deploy to Rill Cloud. + +To apply that theme to a dashboard, add `default_theme: ` to the yaml file for that dashboard. Alternatively, you can add this to the end of the URL in your browser: `?theme=` + + +## Properties + +### `type` + +_[string]_ - Refers to the resource type and must be `theme` _(required)_ + +### `colors` + +_[object]_ - Used to override the dashboard colors. Either primary or secondary color must be provided. _(required)_ + + - **`primary`** - _[string]_ - Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). + + - **`secondary`** - _[string]_ - Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. + +## Common Properties + +### `name` + +_[string]_ - Name is usually inferred from the filename, but can be specified manually. + +### `refs` + +_[array of string]_ - List of resource references + +### `dev` + +_[object]_ - Overrides any properties in development environment. + +### `prod` + +_[object]_ - Overrides any properties in production environment. + +## Examples + +```yaml +# Example: You can copy this directly into your .yaml file +type: theme +colors: + primary: plum + secondary: violet +``` \ No newline at end of file From 3ff888e53e78196cc9fe951cf3fe341fc26c2fcc Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 4 Aug 2025 12:58:38 -0600 Subject: [PATCH 02/32] remove --- test-output/theme.md | 52 -------------------------------------------- 1 file changed, 52 deletions(-) delete mode 100644 test-output/theme.md diff --git a/test-output/theme.md b/test-output/theme.md deleted file mode 100644 index 35d35542419..00000000000 --- a/test-output/theme.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -note: GENERATED. DO NOT EDIT. -title: Theme YAML -sidebar_position: 39 ---- - -In your Rill project directory, create a `.yaml` file in any directory containing `type: theme`. Rill will automatically ingest the theme next time you run `rill start` or deploy to Rill Cloud. - -To apply that theme to a dashboard, add `default_theme: ` to the yaml file for that dashboard. Alternatively, you can add this to the end of the URL in your browser: `?theme=` - - -## Properties - -### `type` - -_[string]_ - Refers to the resource type and must be `theme` _(required)_ - -### `colors` - -_[object]_ - Used to override the dashboard colors. Either primary or secondary color must be provided. _(required)_ - - - **`primary`** - _[string]_ - Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). - - - **`secondary`** - _[string]_ - Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. - -## Common Properties - -### `name` - -_[string]_ - Name is usually inferred from the filename, but can be specified manually. - -### `refs` - -_[array of string]_ - List of resource references - -### `dev` - -_[object]_ - Overrides any properties in development environment. - -### `prod` - -_[object]_ - Overrides any properties in production environment. - -## Examples - -```yaml -# Example: You can copy this directly into your .yaml file -type: theme -colors: - primary: plum - secondary: violet -``` \ No newline at end of file From a381593b90d8b74cf07b1f6d7afe37f13f5f06ca Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 4 Aug 2025 13:29:14 -0600 Subject: [PATCH 03/32] separated YAML schema --- cli/cmd/docs/generate_project.go | 62 +- .../parser/schema/advanced-model.schema.yaml | 523 +++++ runtime/parser/schema/alert.schema.yaml | 282 +++ runtime/parser/schema/api.schema.yaml | 237 +++ runtime/parser/schema/canvas.schema.yaml | 297 +++ runtime/parser/schema/component.schema.yaml | 71 + runtime/parser/schema/connector.schema.yaml | 636 ++++++ runtime/parser/schema/explore.schema.yaml | 285 +++ .../parser/schema/metrics_view.schema.yaml | 324 +++ runtime/parser/schema/project.schema.yaml | 1739 +---------------- runtime/parser/schema/theme.schema.yaml | 68 + 11 files changed, 2776 insertions(+), 1748 deletions(-) create mode 100644 runtime/parser/schema/advanced-model.schema.yaml create mode 100644 runtime/parser/schema/alert.schema.yaml create mode 100644 runtime/parser/schema/api.schema.yaml create mode 100644 runtime/parser/schema/canvas.schema.yaml create mode 100644 runtime/parser/schema/component.schema.yaml create mode 100644 runtime/parser/schema/connector.schema.yaml create mode 100644 runtime/parser/schema/explore.schema.yaml create mode 100644 runtime/parser/schema/metrics_view.schema.yaml create mode 100644 runtime/parser/schema/theme.schema.yaml diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index 84a007457ed..7d5a3560748 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -147,26 +147,52 @@ func resolveRefsYAML(node, root *yaml.Node) error { keyNode := node.Content[i] valNode := node.Content[i+1] - if keyNode.Value == "$ref" && valNode.Kind == yaml.ScalarNode && strings.HasPrefix(valNode.Value, "#/") { - // Resolve local reference - ptrPath := strings.TrimPrefix(valNode.Value, "#/") - resolved, err := resolveYAMLPointer(root, ptrPath) - if err != nil { - return fmt.Errorf("resolve $ref %q: %w", valNode.Value, err) - } + if keyNode.Value == "$ref" && valNode.Kind == yaml.ScalarNode { + if strings.HasPrefix(valNode.Value, "#/") { + // Resolve local reference + ptrPath := strings.TrimPrefix(valNode.Value, "#/") + resolved, err := resolveYAMLPointer(root, ptrPath) + if err != nil { + return fmt.Errorf("resolve $ref %q: %w", valNode.Value, err) + } - // Replace the entire mapping with the resolved content - // First, remove $ref entry - node.Content = append(node.Content[:i], node.Content[i+2:]...) - // Then merge resolved content into current node - if resolved.Kind == yaml.MappingNode { - // Insert resolved mapping node's content at current position - node.Content = append(resolved.Content, node.Content...) - } else { - return fmt.Errorf("$ref does not point to a mapping node") + // Replace the entire mapping with the resolved content + // First, remove $ref entry + node.Content = append(node.Content[:i], node.Content[i+2:]...) + // Then merge resolved content into current node + if resolved.Kind == yaml.MappingNode { + // Insert resolved mapping node's content at current position + node.Content = append(resolved.Content, node.Content...) + } else { + return fmt.Errorf("$ref does not point to a mapping node") + } + // We modified Content length; restart loop + return resolveRefsYAML(node, root) + } else if strings.HasSuffix(valNode.Value, ".yaml#") { + // Resolve external file reference + fileName := strings.TrimSuffix(valNode.Value, "#") + // Remove quotes if present + fileName = strings.Trim(fileName, "'\"") + + // Load the external schema file + externalSchema, err := parseSchemaYAML("runtime/parser/schema/" + fileName) + if err != nil { + return fmt.Errorf("failed to load external schema %q: %w", fileName, err) + } + + // Replace the entire mapping with the external schema content + // First, remove $ref entry + node.Content = append(node.Content[:i], node.Content[i+2:]...) + // Then merge external schema content into current node + if externalSchema.Kind == yaml.MappingNode { + // Insert external schema's content at current position + node.Content = append(externalSchema.Content, node.Content...) + } else { + return fmt.Errorf("external schema %q does not contain a mapping node", fileName) + } + // We modified Content length; restart loop + return resolveRefsYAML(node, root) } - // We modified Content length; restart loop - return resolveRefsYAML(node, root) } if err := resolveRefsYAML(valNode, root); err != nil { return err diff --git a/runtime/parser/schema/advanced-model.schema.yaml b/runtime/parser/schema/advanced-model.schema.yaml new file mode 100644 index 00000000000..e04f3c33825 --- /dev/null +++ b/runtime/parser/schema/advanced-model.schema.yaml @@ -0,0 +1,523 @@ +$schema: 'http://json-schema.org/draft-07/schema#' +$id: advanced-model.schema.yaml +title: Advanced Model YAML +type: object +allOf: + - title: Properties + type: object + properties: + type: + type: string + const: model + description: Refers to the resource type and must be `model` + refresh: + $ref: '#/definitions/schedule_properties' + description: Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying model data + connector: + $ref: 'connector.schema.yaml#' + sql: + type: string + description: Raw SQL query to run against source + timeout: + type: string + description: The maximum time to wait for model ingestion + incremental: + type: boolean + description: whether incremental modeling is required (optional) + change_mode: + type: string + enum: + - reset + - manual + - patch + description: Configure how changes to the model specifications are applied (optional). 'reset' will drop and recreate the model automatically, 'manual' will require a manual full or incremental refresh to apply changes, and 'patch' will switch to the new logic without re-processing historical data (only applies for incremental models). + state: + $ref: '#/definitions/data_properties' + description: Refers to the explicitly defined state of your model, cannot be used with partitions (optional) + partitions: + $ref: '#/definitions/data_properties' + description: Refers to the how your data is partitioned, cannot be used with state. (optional) + materialize: + type: boolean + description: models will be materialized in olap + partitions_watermark: + type: string + description: Refers to a customizable timestamp that can be set to check if an object has been updated (optional). + partitions_concurrency: + type: integer + description: Refers to the number of concurrent partitions that can be read at the same time (optional). + stage: + type: object + properties: + connector: + type: string + description: Refers to the connector type for the staging table + required: + - connector + description: in the case of staging models, where an input source does not support direct write to the output and a staging table is required + additionalProperties: true + output: + type: object + description: to define the properties of output + properties: + table: + type: string + description: Name of the output table. If not specified, the model name is used. + materialize: + type: boolean + description: Whether to materialize the model as a table or view + connector: + type: string + description: Refers to the connector type for the output table. Can be `clickhouse` or `duckdb` and their named connector + incremental_strategy: + type: string + enum: + - append + - merge + - partition_overwrite + description: Strategy to use for incremental updates. Can be 'append', 'merge' or 'partition_overwrite' + unique_key: + type: array + items: + type: string + description: List of columns that uniquely identify a row for merge strategy + partition_by: + type: string + description: Column or expression to partition the table by + allOf: + - if: + title: Additional properties for `output` when `connector` is `clickhouse` + properties: + connector: + const: clickhouse + required: + - connector + then: + properties: + type: + type: string + description: Type to materialize the model into. Can be 'TABLE', 'VIEW' or 'DICTIONARY' + enum: + - TABLE + - VIEW + - DICTIONARY + columns: + type: string + description: Column names and types. Can also include indexes. If unspecified, detected from the query. + engine_full: + type: string + description: Full engine definition in SQL format. Can include partition keys, order, TTL, etc. + engine: + type: string + description: Table engine to use. Default is MergeTree + order_by: + type: string + description: ORDER BY clause. + partition_by: + type: string + description: Partition BY clause. + primary_key: + type: string + description: PRIMARY KEY clause. + sample_by: + type: string + description: SAMPLE BY clause. + ttl: + type: string + description: TTL settings for the table or columns. + table_settings: + type: string + description: Table-specific settings. + query_settings: + type: string + description: Settings used in insert/create table as select queries. + distributed_settings: + type: string + description: Settings for distributed table. + distributed_sharding_key: + type: string + description: Sharding key for distributed table. + dictionary_source_user: + type: string + description: User for accessing the source dictionary table (used if type is DICTIONARY). + dictionary_source_password: + type: string + description: Password for the dictionary source user. + required: + - type + - sql + - $ref: '#/definitions/common_properties' + - type: object + allOf: + - if: + title: Additional properties when `connector` is `athena` or [named connector](./connector.md#name) for athena + properties: + connector: + const: athena + required: + - connector + then: + $ref: '#/definitions/model/definitions/athena' + - if: + title: Additional properties when `connector` is `azure` or [named connector](./connector.md#name) of azure + properties: + connector: + const: azure + required: + - connector + then: + $ref: '#/definitions/model/definitions/azure' + - if: + title: Additional properties when `connector` is `bigquery` or [named connector](./connector.md#name) of bigquery + properties: + connector: + const: bigquery + required: + - connector + then: + $ref: '#/definitions/model/definitions/bigquery' + - if: + title: Additional properties when `connector` is `duckdb` or [named connector](./connector.md#name) of duckdb + properties: + connector: + const: duckdb + required: + - connector + then: + $ref: '#/definitions/model/definitions/duckdb' + - if: + title: Additional properties when `connector` is `gcs` or [named connector](./connector.md#name) of gcs + properties: + connector: + const: gcs + required: + - connector + then: + $ref: '#/definitions/model/definitions/gcs' + - if: + title: Additional properties when `connector` is `local_file` or [named connector](./connector.md#name) of local_file + properties: + connector: + const: local_file + required: + - connector + then: + $ref: '#/definitions/model/definitions/local_file' + - if: + title: Additional properties when `connector` is `redshift` or [named connector](./connector.md#name) of redshift + properties: + connector: + const: redshift + required: + - connector + then: + $ref: '#/definitions/model/definitions/redshift' + - if: + title: Additional properties when `connector` is `s3` or [named connector](./connector.md#name) of s3 + properties: + connector: + const: s3 + required: + - connector + then: + $ref: '#/definitions/model/definitions/s3' + - if: + title: Additional properties when `connector` is `salesforce` or [named connector](./connector.md#name) of salesforce + properties: + connector: + const: salesforce + required: + - connector + then: + $ref: '#/definitions/model/definitions/salesforce' +definitions: + schedule_properties: + type: object + properties: + cron: + type: string + description: A cron expression that defines the execution schedule + time_zone: + type: string + description: Time zone to interpret the schedule in (e.g., 'UTC', 'America/Los_Angeles'). + disable: + type: boolean + description: 'If true, disables the resource without deleting it.' + ref_update: + type: boolean + description: 'If true, allows the resource to run when a dependency updates.' + run_in_dev: + type: boolean + description: 'If true, allows the schedule to run in development mode.' + data_properties: + oneOf: + - title: SQL Query + type: object + description: Executes a raw SQL query against the project's data models. + properties: + sql: + type: string + description: Raw SQL query to run against existing models in the project. + connector: + type: string + description: specifies the connector to use when running SQL or glob queries. + required: + - sql + - title: Metrics View Query + type: object + description: Executes a SQL query that targets a defined metrics view. + properties: + metrics_sql: + type: string + description: SQL query that targets a metrics view in the project + required: + - metrics_sql + - title: Custom API Call + type: object + description: Calls a custom API defined in the project to compute data. + properties: + api: + type: string + description: Name of a custom API defined in the project. + args: + type: object + description: Arguments to pass to the custom API. + additionalProperties: true + required: + - api + - title: File Glob Query + type: object + description: Uses a file-matching pattern (glob) to query data from a connector. + properties: + glob: + description: Defines the file path or pattern to query from the specified connector. + anyOf: + - type: string + description: A simple file path/glob pattern as a string. + - type: object + description: An object-based configuration for specifying a file path/glob pattern with advanced options. + additionalProperties: true + connector: + type: string + description: Specifies the connector to use with the glob input. + required: + - glob + - title: Resource Status Check + type: object + description: Uses the status of a resource as data. + properties: + resource_status: + type: object + description: Based on resource status + properties: + where_error: + type: boolean + description: Indicates whether the condition should trigger when the resource is in an error state. + additionalProperties: true + required: + - resource_status + common_properties: + type: object + title: "Common Properties" + properties: + name: + type: string + description: Name is usually inferred from the filename, but can be specified manually. + refs: + type: array + description: 'List of resource references' + items: + type: string + description: A string reference like `` or ``. + dev: + type: object + description: Overrides any properties in development environment. + prod: + type: object + description: Overrides any properties in production environment. + model: + definitions: + athena: + type: object + properties: + output_location: + type: string + description: Output location for query results in S3. + workgroup: + type: string + description: AWS Athena workgroup to use for queries. + region: + type: string + description: AWS region to connect to Athena and the output location. + azure: + type: object + properties: + path: + type: string + description: Path to the source + account: + type: string + description: Account identifier + uri: + type: string + description: Source URI + extract: + type: object + description: Arbitrary key-value pairs for extraction settings + additionalProperties: true + glob: + type: object + description: Settings related to glob file matching. + properties: + max_total_size: + type: integer + description: Maximum total size (in bytes) matched by glob + max_objects_matched: + type: integer + description: Maximum number of objects matched by glob + max_objects_listed: + type: integer + description: Maximum number of objects listed in glob + page_size: + type: integer + description: Page size for glob listing + batch_size: + type: string + description: 'Size of a batch (e.g., ''100MB'')' + bigquery: + type: object + properties: + project_id: + type: string + description: ID of the BigQuery project. + duckdb: + type: object + properties: + path: + type: string + description: Path to the data source. + format: + type: string + description: 'Format of the data source (e.g., csv, json, parquet).' + pre_exec: + type: string + description: 'refers to SQL queries to run before the main query, available for DuckDB-based models. _(optional)_. Ensure `pre_exec` queries are idempotent. Use `IF NOT EXISTS` statements when applicable.' + post_exec: + type: string + description: 'refers to a SQL query that is run after the main query, available for DuckDB-based models. _(optional)_. Ensure `post_exec` queries are idempotent. Use `IF EXISTS` statements when applicable.' + examples: + - pre_exec: ATTACH IF NOT EXISTS 'dbname=postgres host=localhost port=5432 user=postgres password=postgres' AS postgres_db (TYPE POSTGRES); + sql: SELECT * FROM postgres_query('postgres_db', 'SELECT * FROM USERS') + post_exec: DETACH DATABASE IF EXISTS postgres_db + gcs: + type: object + properties: + path: + type: string + description: Path to the source + uri: + type: string + description: Source URI + extract: + type: object + description: key-value pairs for extraction settings + additionalProperties: true + glob: + type: object + description: Settings related to glob file matching. + properties: + max_total_size: + type: integer + description: Maximum total size (in bytes) matched by glob + max_objects_matched: + type: integer + description: Maximum number of objects matched by glob + max_objects_listed: + type: integer + description: Maximum number of objects listed in glob + page_size: + type: integer + description: Page size for glob listing + batch_size: + type: string + description: 'Size of a batch (e.g., ''100MB'')' + local_file: + type: object + properties: + path: + type: string + description: Path to the data source. + format: + type: string + description: 'Format of the data source (e.g., csv, json, parquet).' + redshift: + type: object + properties: + output_location: + type: string + description: S3 location where query results are stored. + workgroup: + type: string + description: Redshift Serverless workgroup to use. + database: + type: string + description: Name of the Redshift database. + cluster_identifier: + type: string + description: Identifier of the Redshift cluster. + role_arn: + type: string + description: ARN of the IAM role to assume for Redshift access. + region: + type: string + description: AWS region of the Redshift deployment. + s3: + type: object + properties: + region: + type: string + description: AWS region + endpoint: + type: string + description: AWS Endpoint + path: + type: string + description: Path to the source + uri: + type: string + description: Source URI + extract: + type: object + description: key-value pairs for extraction settings + additionalProperties: true + glob: + type: object + description: Settings related to glob file matching. + properties: + max_total_size: + type: integer + description: Maximum total size (in bytes) matched by glob + max_objects_matched: + type: integer + description: Maximum number of objects matched by glob + max_objects_listed: + type: integer + description: Maximum number of objects listed in glob + page_size: + type: integer + description: Page size for glob listing + batch_size: + type: string + description: 'Size of a batch (e.g., ''100MB'')' + salesforce: + type: object + properties: + soql: + type: string + description: SOQL query to execute against the Salesforce instance. + sobject: + type: string + description: Salesforce object (e.g., Account, Contact) targeted by the query. + queryAll: + type: boolean + description: Whether to include deleted and archived records in the query (uses queryAll API). \ No newline at end of file diff --git a/runtime/parser/schema/alert.schema.yaml b/runtime/parser/schema/alert.schema.yaml new file mode 100644 index 00000000000..9efb755c557 --- /dev/null +++ b/runtime/parser/schema/alert.schema.yaml @@ -0,0 +1,282 @@ +$schema: 'http://json-schema.org/draft-07/schema#' +$id: alert.schema.yaml +title: Alert YAML +type: object +description: Along with alertings at the dashboard level and can be created via the UI, there might be more extensive alerting that you might want to develop and can be done so the an alert.yaml. When creating an alert via a YAML file, you'll see this denoted in the UI as `Created through code`. +examples: + - # Example: To send alert when data lags by more than 1 day to slack channel #rill-cloud-alerts + type: alert + display_name: Data lags by more than 1 day + # Check the alert every hour. + refresh: + cron: 0 * * * * + # Query that returns non-empty results if the metrics lag by more than 1 day. + data: + sql: |- + SELECT * + FROM + ( + SELECT MAX(event_time) AS max_time + FROM rill_metrics_model + ) + WHERE max_time < NOW() - INTERVAL '1 day' + # Send notifications in Slack. + notify: + slack: + channels: + - '#rill-cloud-alerts' +allOf: + - title: Properties + type: object + properties: + type: + type: string + const: alert + description: Refers to the resource type and must be `alert` + display_name: + type: string + description: Refers to the display name for the alert + refresh: + $ref: '#/definitions/schedule_properties' + description: Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying data + intervals: + type: object + description: define the interval of the alert to check + properties: + duration: + type: string + description: a valid ISO8601 duration to define the interval duration + limit: + type: integer + description: maximum number of intervals to check for on invocation + minimum: 0 + check_unclosed: + type: boolean + description: 'boolean, whether unclosed intervals should be checked' + watermark: + type: string + enum: + - trigger_time + - inherit + description: Specifies how the watermark is determined for incremental processing. Use 'trigger_time' to set it at runtime or 'inherit' to use the upstream model's watermark. + timeout: + type: string + description: define the timeout of the alert in seconds (optional). + data: + description: Specifies one of the options to retrieve or compute the data used by alert + $ref: '#/definitions/data_properties' + for: + description: "Specifies how user identity or attributes should be evaluated for security policy enforcement." + oneOf: + - type: object + description: Specifies a unique user identifier for applying security policies. + properties: + user_id: + type: string + description: "The unique user ID used to evaluate security policies." + required: + - user_id + additionalProperties: false + - type: object + description: Specifies a user's email address for applying security policies. + properties: + user_email: + type: string + description: "The user's email address used to evaluate security policies." + format: email + required: + - user_email + additionalProperties: false + - type: object + description: Specifies a set of arbitrary user attributes for applying security policies. + properties: + attributes: + type: object + description: A dictionary of user attributes used to evaluate security policies. + additionalProperties: true + required: + - attributes + additionalProperties: false + on_recover: + type: boolean + description: Send an alert when a previously failing alert recovers. Defaults to false. + on_fail: + type: boolean + description: Send an alert when a failure occurs. Defaults to true. + on_error: + type: boolean + description: Send an alert when an error occurs during evaluation. Defaults to false. + renotify: + type: boolean + description: Enable repeated notifications for unresolved alerts. Defaults to false. + renotify_after: + type: string + description: Defines the re-notification interval for the alert (e.g., '10m','24h'), equivalent to snooze duration in UI, defaults to 'Off' + notify: + $ref: '#/definitions/notify_properties' + description: Defines how and where to send notifications. At least one method (email or Slack) is required. + annotations: + type: object + description: Key value pair used for annotations + additionalProperties: + type: string + required: + - type + - refresh + - data + - notify + - $ref: '#/definitions/common_properties' +definitions: + schedule_properties: + type: object + properties: + cron: + type: string + description: A cron expression that defines the execution schedule + time_zone: + type: string + description: Time zone to interpret the schedule in (e.g., 'UTC', 'America/Los_Angeles'). + disable: + type: boolean + description: 'If true, disables the resource without deleting it.' + ref_update: + type: boolean + description: 'If true, allows the resource to run when a dependency updates.' + run_in_dev: + type: boolean + description: 'If true, allows the schedule to run in development mode.' + data_properties: + oneOf: + - title: SQL Query + type: object + description: Executes a raw SQL query against the project's data models. + properties: + sql: + type: string + description: Raw SQL query to run against existing models in the project. + connector: + type: string + description: specifies the connector to use when running SQL or glob queries. + required: + - sql + - title: Metrics View Query + type: object + description: Executes a SQL query that targets a defined metrics view. + properties: + metrics_sql: + type: string + description: SQL query that targets a metrics view in the project + required: + - metrics_sql + - title: Custom API Call + type: object + description: Calls a custom API defined in the project to compute data. + properties: + api: + type: string + description: Name of a custom API defined in the project. + args: + type: object + description: Arguments to pass to the custom API. + additionalProperties: true + required: + - api + - title: File Glob Query + type: object + description: Uses a file-matching pattern (glob) to query data from a connector. + properties: + glob: + description: Defines the file path or pattern to query from the specified connector. + anyOf: + - type: string + description: A simple file path/glob pattern as a string. + - type: object + description: An object-based configuration for specifying a file path/glob pattern with advanced options. + additionalProperties: true + connector: + type: string + description: Specifies the connector to use with the glob input. + required: + - glob + - title: Resource Status Check + type: object + description: Uses the status of a resource as data. + properties: + resource_status: + type: object + description: Based on resource status + properties: + where_error: + type: boolean + description: Indicates whether the condition should trigger when the resource is in an error state. + additionalProperties: true + required: + - resource_status + notify_properties: + type: object + properties: + email: + type: object + description: Send notifications via email. + properties: + recipients: + type: array + description: An array of email addresses to notify. + items: + type: string + minItems: 1 + required: + - recipients + slack: + type: object + description: Send notifications via Slack. + properties: + users: + type: array + description: An array of Slack user IDs to notify. + items: + type: string + minItems: 1 + channels: + type: array + description: An array of Slack channel IDs to notify. + items: + type: string + minItems: 1 + webhooks: + type: array + description: An array of Slack webhook URLs to send notifications to. + items: + type: string + minItems: 1 + anyOf: + - required: + - channels + - required: + - users + - required: + - webhooks + anyOf: + - required: + - slack + - required: + - email + common_properties: + type: object + title: "Common Properties" + properties: + name: + type: string + description: Name is usually inferred from the filename, but can be specified manually. + refs: + type: array + description: 'List of resource references' + items: + type: string + description: A string reference like `` or ``. + dev: + type: object + description: Overrides any properties in development environment. + prod: + type: object + description: Overrides any properties in production environment. \ No newline at end of file diff --git a/runtime/parser/schema/api.schema.yaml b/runtime/parser/schema/api.schema.yaml new file mode 100644 index 00000000000..b7c1c39d3be --- /dev/null +++ b/runtime/parser/schema/api.schema.yaml @@ -0,0 +1,237 @@ +$schema: 'http://json-schema.org/draft-07/schema#' +$id: api.schema.yaml +title: API YAML +type: object +description: In your Rill project directory, create a new file name `.yaml` in the `apis` directory containing a custom API definition. See comprehensive documentation on how to define and use [custom APIs](/integrate/custom-apis/index.md) +examples: + - # Example: This api returns the top 10 authors by net line changes since the specified date provided in the arguments. + type: api + name: metrics_view_api + metrics_sql: |- + SELECT author_name, net_line_changes + FROM advanced_metrics_view + where author_date > '{{ .args.date }}' + order by net_line_changes DESC + limit 10 +allOf: + - title: Properties + type: object + properties: + type: + type: string + const: api + description: Refers to the resource type and must be `api` + openapi: + type: object + description: OpenAPI specification for the API endpoint + properties: + summary: + type: string + description: A brief description of what the API endpoint does + parameters: + type: array + description: List of parameters that the API endpoint accepts + items: + type: object + additionalProperties: true + request_schema: + type: object + description: JSON schema for the request body (use nested YAML instead of a JSON string) + additionalProperties: true + response_schema: + type: object + description: JSON schema for the response body (use nested YAML instead of a JSON string) + additionalProperties: true + security: + $ref: '#/definitions/security_policy_properties' + skip_nested_security: + type: boolean + description: Flag to control security inheritance + allOf: + - $ref: '#/definitions/data_properties' + required: + - type + - $ref: '#/definitions/common_properties' +definitions: + data_properties: + oneOf: + - title: SQL Query + type: object + description: Executes a raw SQL query against the project's data models. + properties: + sql: + type: string + description: Raw SQL query to run against existing models in the project. + connector: + type: string + description: specifies the connector to use when running SQL or glob queries. + required: + - sql + - title: Metrics View Query + type: object + description: Executes a SQL query that targets a defined metrics view. + properties: + metrics_sql: + type: string + description: SQL query that targets a metrics view in the project + required: + - metrics_sql + - title: Custom API Call + type: object + description: Calls a custom API defined in the project to compute data. + properties: + api: + type: string + description: Name of a custom API defined in the project. + args: + type: object + description: Arguments to pass to the custom API. + additionalProperties: true + required: + - api + - title: File Glob Query + type: object + description: Uses a file-matching pattern (glob) to query data from a connector. + properties: + glob: + description: Defines the file path or pattern to query from the specified connector. + anyOf: + - type: string + description: A simple file path/glob pattern as a string. + - type: object + description: An object-based configuration for specifying a file path/glob pattern with advanced options. + additionalProperties: true + connector: + type: string + description: Specifies the connector to use with the glob input. + required: + - glob + - title: Resource Status Check + type: object + description: Uses the status of a resource as data. + properties: + resource_status: + type: object + description: Based on resource status + properties: + where_error: + type: boolean + description: Indicates whether the condition should trigger when the resource is in an error state. + additionalProperties: true + required: + - resource_status + security_policy_properties: + type: object + description: Defines security rules and access control policies for resources + properties: + access: + oneOf: + - type: string + description: SQL expression that evaluates to a boolean to determine access + - type: boolean + description: Direct boolean value to allow or deny access + description: Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + row_filter: + type: string + description: SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause + include: + type: array + description: List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded + items: + type: object + properties: + if: + type: string + description: Expression to decide if the column should be included or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean + names: + anyOf: + - type: array + description: List of specific field names to include + items: + type: string + - type: string + description: Wildcard '*' to include all fields + enum: + - '*' + description: List of fields to include. Should match the name of one of the dashboard's dimensions or measures + required: + - if + - names + exclude: + type: array + description: List of dimension or measure names to exclude from the dashboard. If exclude is defined all other dimensions and measures are included + items: + type: object + properties: + if: + type: string + description: Expression to decide if the column should be excluded or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean + names: + anyOf: + - type: array + description: List of specific field names to exclude + items: + type: string + - type: string + description: Wildcard '*' to exclude all fields + enum: + - '*' + description: List of fields to exclude. Should match the name of one of the dashboard's dimensions or measures + required: + - if + - names + rules: + type: array + description: List of detailed security rules that can be used to define complex access control policies + items: + type: object + description: Individual security rule definition + properties: + type: + type: string + enum: + - access + - field_access + - row_filter + description: Type of security rule - access (overall access), field_access (field-level access), or row_filter (row-level filtering) + action: + type: string + enum: + - allow + - deny + description: Whether to allow or deny access for this rule + if: + type: string + description: Conditional expression that determines when this rule applies. Must be a valid SQL expression that evaluates to a boolean + names: + type: array + items: + type: string + description: List of field names this rule applies to (for field_access type rules) + all: + type: boolean + description: When true, applies the rule to all fields (for field_access type rules) + sql: + type: string + description: SQL expression for row filtering (for row_filter type rules) + required: + - type + common_properties: + type: object + title: "Common Properties" + properties: + name: + type: string + description: Name is usually inferred from the filename, but can be specified manually. + refs: + type: array + description: 'List of resource references' + items: + type: string + description: A string reference like `` or ``. + dev: + type: object + description: Overrides any properties in development environment. + prod: + type: object + description: Overrides any properties in production environment. \ No newline at end of file diff --git a/runtime/parser/schema/canvas.schema.yaml b/runtime/parser/schema/canvas.schema.yaml new file mode 100644 index 00000000000..e5b03211016 --- /dev/null +++ b/runtime/parser/schema/canvas.schema.yaml @@ -0,0 +1,297 @@ +$schema: 'http://json-schema.org/draft-07/schema#' +$id: canvas.schema.yaml +title: Canvas YAML +type: object +description: In your Rill project directory, create a explore dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. +allOf: + - title: Properties + type: object + properties: + type: + type: string + const: canvas + description: Refers to the resource type and must be `canvas` + display_name: + type: string + description: Refers to the display name for the canvas + banner: + type: string + description: Refers to the custom banner displayed at the header of an Canvas dashboard + max_width: + type: integer + description: Max width in pixels of the canvas + minimum: 0 + gap_x: + type: integer + description: Horizontal gap in pixels of the canvas + minimum: 0 + gap_y: + type: integer + description: Vertical gap in pixels of the canvas + minimum: 0 + theme: + oneOf: + - type: string + description: Name of an existing theme to apply to the dashboard + - $ref: '#/definitions/theme/definitions/theme_properties' + description: Inline theme configuration. + description: Name of the theme to use. Only one of theme and embedded_theme can be set. + allow_custom_time_range: + type: boolean + description: Defaults to true, when set to false it will hide the ability to set a custom time range for the user. + time_ranges: + type: array + description: Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' + items: + $ref: '#/definitions/explore_time_range_properties' + time_zones: + type: array + description: Refers to the time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) + items: + type: string + filters: + type: object + description: Indicates if filters should be enabled for the canvas. + properties: + enable: + type: boolean + description: Toggles filtering functionality for the canvas dashboard. + additionalProperties: false + defaults: + type: object + description: Preset UI state to show by default + properties: + time_range: + type: string + description: Default time range to display when the dashboard loads + comparison_mode: + type: string + description: Default comparison mode for metrics (none, time, or dimension) + comparison_dimension: + type: string + description: Default dimension to use for comparison when comparison_mode is 'dimension' + additionalProperties: false + variables: + type: array + description: Variables that can be used in the canvas + items: + $ref: '#/definitions/component_variable_properties' + rows: + type: array + description: Refers to all of the rows displayed on the Canvas + items: + type: object + properties: + height: + type: string + description: Height of the row in px + items: + type: array + description: List of components to display in the row + items: + type: object + properties: + component: + type: string + description: Name of the component to display + width: + type: + - string + - integer + description: Width of the component (can be a number or string with unit) + additionalProperties: true + additionalProperties: false + security: + $ref: '#/definitions/security_policy_properties' + description: Security rules to apply for access to the canvas + required: + - type + - rows + - $ref: '#/definitions/common_properties' +definitions: + explore_time_range_properties: + oneOf: + - type: string + description: a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection + - type: object + description: Object containing time range and comparison configuration + properties: + range: + type: string + description: a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection + comparison_offsets: + type: array + description: list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + items: + oneOf: + - type: string + description: Offset string only (range is inferred) + - type: object + description: Object containing offset and range configuration for time comparison + properties: + offset: + type: string + description: Time offset for comparison (e.g., 'P1D' for one day ago) + range: + type: string + description: Custom time range for comparison period + additionalProperties: false + required: + - range + additionalProperties: false + component_variable_properties: + type: object + properties: + name: + type: string + description: Unique identifier for the variable + type: + type: string + description: Data type of the variable (e.g., string, number, boolean) + value: + description: Default value for the variable. Can be any valid JSON value type + type: + - string + - number + - boolean + - object + - array + required: + - name + - type + additionalProperties: false + security_policy_properties: + type: object + description: Defines security rules and access control policies for resources + properties: + access: + oneOf: + - type: string + description: SQL expression that evaluates to a boolean to determine access + - type: boolean + description: Direct boolean value to allow or deny access + description: Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + row_filter: + type: string + description: SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause + include: + type: array + description: List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded + items: + type: object + properties: + if: + type: string + description: Expression to decide if the column should be included or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean + names: + anyOf: + - type: array + description: List of specific field names to include + items: + type: string + - type: string + description: Wildcard '*' to include all fields + enum: + - '*' + description: List of fields to include. Should match the name of one of the dashboard's dimensions or measures + required: + - if + - names + exclude: + type: array + description: List of dimension or measure names to exclude from the dashboard. If exclude is defined all other dimensions and measures are included + items: + type: object + properties: + if: + type: string + description: Expression to decide if the column should be excluded or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean + names: + anyOf: + - type: array + description: List of specific field names to exclude + items: + type: string + - type: string + description: Wildcard '*' to exclude all fields + enum: + - '*' + description: List of fields to exclude. Should match the name of one of the dashboard's dimensions or measures + required: + - if + - names + rules: + type: array + description: List of detailed security rules that can be used to define complex access control policies + items: + type: object + description: Individual security rule definition + properties: + type: + type: string + enum: + - access + - field_access + - row_filter + description: Type of security rule - access (overall access), field_access (field-level access), or row_filter (row-level filtering) + action: + type: string + enum: + - allow + - deny + description: Whether to allow or deny access for this rule + if: + type: string + description: Conditional expression that determines when this rule applies. Must be a valid SQL expression that evaluates to a boolean + names: + type: array + items: + type: string + description: List of field names this rule applies to (for field_access type rules) + all: + type: boolean + description: When true, applies the rule to all fields (for field_access type rules) + sql: + type: string + description: SQL expression for row filtering (for row_filter type rules) + required: + - type + common_properties: + type: object + title: "Common Properties" + properties: + name: + type: string + description: Name is usually inferred from the filename, but can be specified manually. + refs: + type: array + description: 'List of resource references' + items: + type: string + description: A string reference like `` or ``. + dev: + type: object + description: Overrides any properties in development environment. + prod: + type: object + description: Overrides any properties in production environment. + theme: + definitions: + theme_properties: + type: object + properties: + colors: + type: object + description: Used to override the dashboard colors. Either primary or secondary color must be provided. + properties: + primary: + type: string + description: Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). + secondary: + type: string + description: Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. + anyOf: + - required: + - primary + - required: + - secondary \ No newline at end of file diff --git a/runtime/parser/schema/component.schema.yaml b/runtime/parser/schema/component.schema.yaml new file mode 100644 index 00000000000..e5c71366692 --- /dev/null +++ b/runtime/parser/schema/component.schema.yaml @@ -0,0 +1,71 @@ +$schema: 'http://json-schema.org/draft-07/schema#' +$id: component.schema.yaml +title: Component YAML +type: object +description: Defines a reusable dashboard component that can be embedded in canvas dashboards +allOf: + - title: Properties + type: object + properties: + type: + type: string + const: component + description: Refers to the resource type and must be `component` + display_name: + type: string + description: Refers to the display name for the component + description: + type: string + description: Detailed description of the component's purpose and functionality + input: + type: array + description: List of input variables that can be passed to the component + items: + $ref: '#/definitions/component_variable_properties' + output: + description: Output variable that the component produces + $ref: '#/definitions/component_variable_properties' + required: + - type + - $ref: '#/definitions/common_properties' +definitions: + component_variable_properties: + type: object + properties: + name: + type: string + description: Unique identifier for the variable + type: + type: string + description: Data type of the variable (e.g., string, number, boolean) + value: + description: Default value for the variable. Can be any valid JSON value type + type: + - string + - number + - boolean + - object + - array + required: + - name + - type + additionalProperties: false + common_properties: + type: object + title: "Common Properties" + properties: + name: + type: string + description: Name is usually inferred from the filename, but can be specified manually. + refs: + type: array + description: 'List of resource references' + items: + type: string + description: A string reference like `` or ``. + dev: + type: object + description: Overrides any properties in development environment. + prod: + type: object + description: Overrides any properties in production environment. \ No newline at end of file diff --git a/runtime/parser/schema/connector.schema.yaml b/runtime/parser/schema/connector.schema.yaml new file mode 100644 index 00000000000..e33442cc0d4 --- /dev/null +++ b/runtime/parser/schema/connector.schema.yaml @@ -0,0 +1,636 @@ +$schema: 'http://json-schema.org/draft-07/schema#' +$id: connector.schema.yaml +title: Connector YAML +type: object +description: | + Connector YAML files define how Rill connects to external data sources and OLAP engines. Each connector specifies a driver type and its required connection parameters. +allOf: + - title: Properties + type: object + properties: + type: + type: string + const: connector + description: Refers to the resource type and must be `connector` + driver: + type: string + description: The type of connector, see [available connectors](#available-connector-types) (required) + required: + - type + - driver + - $ref: '#/definitions/common_properties' + - oneOf: + - $ref: '#/definitions/athena' + - $ref: '#/definitions/azure' + - $ref: '#/definitions/bigquery' + - $ref: '#/definitions/clickhouse' + - $ref: '#/definitions/druid' + - $ref: '#/definitions/duckdb' + - $ref: '#/definitions/gcs' + - $ref: '#/definitions/https' + - $ref: '#/definitions/local_file' + - $ref: '#/definitions/motherduck' + - $ref: '#/definitions/mysql' + - $ref: '#/definitions/pinot' + - $ref: '#/definitions/postgres' + - $ref: '#/definitions/redshift' + - $ref: '#/definitions/s3' + - $ref: '#/definitions/salesforce' + - $ref: '#/definitions/slack' + - $ref: '#/definitions/snowflake' + - $ref: '#/definitions/sqlite' +definitions: + athena: + type: object + title: athena + description: Configuration properties specific to the athena + properties: + driver: + type: string + description: Refers to the driver type and must be driver `athena` + const: athena + aws_access_key_id: + type: string + description: AWS Access Key ID used for authentication. Required when using static credentials directly or as base credentials for assuming a role. + aws_secret_access_key: + type: string + description: AWS Secret Access Key paired with the Access Key ID. Required when using static credentials directly or as base credentials for assuming a role. + aws_access_token: + type: string + description: AWS session token used with temporary credentials. Required only if the Access Key and Secret Key are part of a temporary session credentials. + role_arn: + type: string + description: ARN of the IAM role to assume. When specified, the SDK uses the base credentials to call STS AssumeRole and obtain temporary credentials scoped to this role. + role_session_name: + type: string + description: Session name to associate with the STS AssumeRole session. Used only if 'role_arn' is specified. Useful for identifying and auditing the session. + external_id: + type: string + description: External ID required by some roles when assuming them, typically for cross-account access. Used only if 'role_arn' is specified and the role's trust policy requires it. + workgroup: + type: string + description: Athena workgroup to use for query execution. Defaults to 'primary' if not specified. + output_location: + type: string + description: S3 URI where Athena query results should be stored (e.g., s3://your-bucket/athena/results/). Optional if the selected workgroup has a default result configuration. + aws_region: + type: string + description: AWS region where Athena and the result S3 bucket are located (e.g., us-east-1). Defaults to 'us-east-1' if not specified. + allow_host_access: + type: boolean + description: Allow the Athena client to access host environment configurations such as environment variables or local AWS credential files. Defaults to true, enabling use of credentials and settings from the host environment unless explicitly disabled. + required: + - driver + azure: + type: object + title: azure + description: Configuration properties specific to the azure + properties: + driver: + type: string + description: Refers to the driver type and must be driver `azure` + const: azure + azure_storage_account: + type: string + description: Azure storage account name + azure_storage_key: + type: string + description: Azure storage access key + azure_storage_sas_token: + type: string + description: Optional azure SAS token for authentication + azure_storage_connection_string: + type: string + description: Optional azure connection string for storage account + azure_storage_bucket: + type: string + description: Name of the Azure Blob Storage container (equivalent to an S3 bucket) + allow_host_access: + type: boolean + description: Allow access to host environment configuration + required: + - driver + - azure_storage_bucket + bigquery: + type: object + title: bigquery + description: Configuration properties specific to the bigquery + properties: + driver: + type: string + description: Refers to the driver type and must be driver `bigquery` + const: bigquery + google_application_credentials: + type: string + description: Raw contents of the Google Cloud service account key (in JSON format) used for authentication. + project_id: + type: string + description: ID of the Google Cloud project to use for BigQuery operations. This can be omitted only if the project ID is included in the service account key. + allow_host_access: + type: boolean + description: Enable the BigQuery client to use credentials from the host environment when no service account JSON is provided. This includes Application Default Credentials from environment variables, local credential files, or the Google Compute Engine metadata server. Defaults to true, allowing seamless authentication in GCP environments. + required: + - driver + clickhouse: + type: object + title: clickhouse + description: Configuration properties specific to the clickhouse + properties: + driver: + type: string + description: Refers to the driver type and must be driver `clickhouse` + const: clickhouse + managed: + type: boolean + description: '`true` means Rill will provision the connector using the default provisioner. `false` disables automatic provisioning.' + mode: + type: string + description: "`read` - Controls the operation mode for the ClickHouse connection. Defaults to 'read' for safe operation with external databases. Set to 'readwrite' to enable model creation and table mutations. Note: When 'managed: true', this is automatically set to 'readwrite'." + dsn: + type: string + description: DSN(Data Source Name) for the ClickHouse connection + username: + type: string + description: Username for authentication + password: + type: string + description: Password for authentication + host: + type: string + description: Host where the ClickHouse instance is running + port: + type: integer + description: Port where the ClickHouse instance is accessible + database: + type: string + description: Name of the ClickHouse database within the cluster + ssl: + type: boolean + description: Indicates whether a secured SSL connection is required + cluster: + type: string + description: 'Cluster name, required for running distributed queries' + log_queries: + type: boolean + description: Controls whether to log raw SQL queries + settings_override: + type: string + description: override the default settings used in queries. example `readonly = 1, session_timezone = 'UTC'` + embed_port: + type: integer + description: Port to run ClickHouse locally (0 for random port) + can_scale_to_zero: + type: boolean + description: Indicates if the database can scale to zero + max_open_conns: + type: integer + description: Maximum number of open connections to the database + max_idle_conns: + type: integer + description: Maximum number of idle connections in the pool + dial_timeout: + type: string + description: Timeout for dialing the ClickHouse server + conn_max_lifetime: + type: string + description: Maximum time a connection may be reused + read_timeout: + type: string + description: Maximum time for a connection to read data + required: + - driver + druid: + type: object + title: druid + description: Configuration properties specific to the druid + properties: + driver: + type: string + description: Refers to the driver type and must be driver `druid` + const: druid + dsn: + type: string + description: Data Source Name (DSN) for connecting to Druid + username: + type: string + description: Username for authenticating with Druid + password: + type: string + description: Password for authenticating with Druid + host: + type: string + description: Hostname of the Druid coordinator or broker + port: + type: integer + description: Port number of the Druid service + ssl: + type: boolean + description: Enable SSL for secure connection + log_queries: + type: boolean + description: Log raw SQL queries sent to Druid + max_open_conns: + type: integer + description: Maximum number of open database connections (0 = default, -1 = unlimited) + skip_version_check: + type: boolean + description: Skip checking Druid version compatibility + required: + - driver + - dsn + duckdb: + type: object + title: duckdb + description: Configuration properties specific to the duckdb + properties: + driver: + type: string + description: Refers to the driver type and must be driver `duckdb` + const: duckdb + pool_size: + type: integer + description: Number of concurrent connections and queries allowed + allow_host_access: + type: boolean + description: Whether access to the local environment and file system is allowed + cpu: + type: integer + description: Number of CPU cores available to the database + memory_limit_gb: + type: integer + description: Amount of memory in GB available to the database + read_write_ratio: + type: number + description: Ratio of resources allocated to the read database; used to divide CPU and memory + init_sql: + type: string + description: is executed during database initialization. + conn_init_sql: + type: string + description: is executed when a new connection is initialized. + secrets: + type: string + description: Comma-separated list of other connector names to create temporary secrets for in DuckDB before executing a model. + log_queries: + type: boolean + description: Whether to log raw SQL queries executed through OLAP + required: + - driver + gcs: + type: object + title: gcs + description: Configuration properties specific to the gcs + properties: + driver: + type: string + description: Refers to the driver type and must be driver `gcs` + const: gcs + google_application_credentials: + type: string + description: Google Cloud credentials JSON string + bucket: + type: string + description: Name of gcs bucket + allow_host_access: + type: boolean + description: Allow access to host environment configuration + key_id: + type: string + description: Optional S3-compatible Key ID when used in compatibility mode + secret: + type: string + description: Optional S3-compatible Secret when used in compatibility mode + required: + - driver + - bucket + https: + type: object + title: https + description: Configuration properties specific to the https + properties: + driver: + type: string + description: Refers to the driver type and must be driver `https` + const: https + path: + type: string + description: The full HTTPS URI to fetch data from + headers: + type: object + description: HTTP headers to include in the request + additionalProperties: + type: string + required: + - driver + - path + local_file: + type: object + title: local_file + description: Configuration properties specific to the local_file + properties: + driver: + type: string + description: Refers to the driver type and must be driver `local_file` + const: local_file + dsn: + type: string + description: Data Source Name (DSN) indicating the file path or location of the local file + allow_host_access: + type: boolean + description: Flag to indicate if access to host-level file paths is permitted + required: + - driver + - dsn + motherduck: + type: object + title: motherduck + description: Configuration properties specific to the motherduck + properties: + driver: + type: string + description: Refers to the driver type and must be driver `motherduck` + const: motherduck + dsn: + type: string + description: Data Source Name (DSN) specifying the MotherDuck connection endpoint + token: + type: string + description: Authentication token for accessing MotherDuck (secret) + required: + - driver + - dsn + - token + mysql: + type: object + title: mysql + description: Configuration properties specific to the mysql + properties: + driver: + type: string + description: Refers to the driver type and must be driver `mysql` + const: mysql + dsn: + type: string + description: DSN(Data Source Name) for the mysql connection + host: + type: string + description: Hostname of the MySQL server + port: + type: integer + description: Port number for the MySQL server + database: + type: string + description: Name of the MySQL database + user: + type: string + description: Username for authentication + password: + type: string + description: Password for authentication + ssl_mode: + type: string + description: SSL mode can be DISABLED, PREFERRED or REQUIRED + required: + - driver + pinot: + type: object + title: pinot + description: Configuration properties specific to the pinot + properties: + driver: + type: string + description: Refers to the driver type and must be driver `pinot` + const: pinot + dsn: + type: string + description: DSN(Data Source Name) for the Pinot connection + username: + type: string + description: Username for authenticating with Pinot + password: + type: string + description: Password for authenticating with Pinot + broker_host: + type: string + description: Hostname of the Pinot broker + broker_port: + type: integer + description: Port number for the Pinot broker + controller_host: + type: string + description: Hostname of the Pinot controller + controller_port: + type: integer + description: Port number for the Pinot controller + ssl: + type: boolean + description: Enable SSL connection to Pinot + log_queries: + type: boolean + description: Log raw SQL queries executed through Pinot + max_open_conns: + type: integer + description: Maximum number of open connections to the Pinot database + required: + - driver + - dsn + - broker_host + - controller_host + postgres: + type: object + title: postgres + description: Configuration properties specific to the postgres + properties: + driver: + type: string + description: Refers to the driver type and must be driver `postgres` + const: postgres + dsn: + type: string + description: DSN(Data Source Name) for the postgres connection + host: + type: string + description: Hostname of the Postgres server + port: + type: string + description: Port number for the Postgres server + dbname: + type: string + description: Name of the Postgres database + user: + type: string + description: Username for authentication + password: + type: string + description: Password for authentication + sslmode: + type: string + description: SSL mode can be disable, allow, prefer or require + required: + - driver + redshift: + type: object + title: redshift + description: Configuration properties specific to the redshift + properties: + driver: + type: string + description: Refers to the driver type and must be driver `redshift` + const: redshift + aws_access_key_id: + type: string + description: AWS Access Key ID used for authenticating with Redshift. + aws_secret_access_key: + type: string + description: AWS Secret Access Key used for authenticating with Redshift. + aws_access_token: + type: string + description: AWS Session Token for temporary credentials (optional). + region: + type: string + description: AWS region where the Redshift cluster or workgroup is hosted (e.g., 'us-east-1'). + database: + type: string + description: Name of the Redshift database to query. + workgroup: + type: string + description: Workgroup name for Redshift Serverless, in case of provisioned Redshift clusters use 'cluster_identifier'. + cluster_identifier: + type: string + description: Cluster identifier for provisioned Redshift clusters, in case of Redshift Serverless use 'workgroup' . + required: + - driver + - aws_access_key_id + - aws_secret_access_key + - database + s3: + type: object + title: s3 + description: Configuration properties specific to the s3 + properties: + driver: + type: string + description: Refers to the driver type and must be driver `s3` + const: s3 + aws_access_key_id: + type: string + description: AWS Access Key ID used for authentication + aws_secret_access_key: + type: string + description: AWS Secret Access Key used for authentication + aws_access_token: + type: string + description: Optional AWS session token for temporary credentials + bucket: + type: string + description: Name of s3 bucket + endpoint: + type: string + description: Optional custom endpoint URL for S3-compatible storage + region: + type: string + description: AWS region of the S3 bucket + allow_host_access: + type: boolean + description: Allow access to host environment configuration + retain_files: + type: boolean + description: Whether to retain intermediate files after processing + required: + - driver + - bucket + salesforce: + type: object + title: salesforce + description: Configuration properties specific to the salesforce + properties: + driver: + type: string + description: Refers to the driver type and must be driver `salesforce` + const: salesforce + username: + type: string + description: Salesforce account username + password: + type: string + description: Salesforce account password (secret) + key: + type: string + description: Authentication key for Salesforce (secret) + endpoint: + type: string + description: Salesforce API endpoint URL + client_id: + type: string + description: Client ID used for Salesforce OAuth authentication + required: + - driver + - username + - endpoint + slack: + type: object + title: slack + description: Configuration properties specific to the slack + properties: + driver: + type: string + description: Refers to the driver type and must be driver `slack` + const: slack + bot_token: + type: string + description: Bot token used for authenticating Slack API requests + required: + - driver + - bot_token + snowflake: + type: object + title: snowflake + description: Configuration properties specific to the snowflake + properties: + driver: + type: string + description: Refers to the driver type and must be driver `snowflake` + const: snowflake + dsn: + type: string + description: DSN (Data Source Name) for the Snowflake connection + parallel_fetch_limit: + type: integer + description: Maximum number of concurrent fetches during query execution + required: + - driver + - dsn + sqlite: + type: object + title: sqlite + description: Configuration properties specific to the sqlite + properties: + driver: + type: string + description: Refers to the driver type and must be driver `sqlite` + const: sqlite + dsn: + type: string + description: DSN(Data Source Name) for the sqlite connection + required: + - driver + - dsn + common_properties: + type: object + title: "Common Properties" + properties: + name: + type: string + description: Name is usually inferred from the filename, but can be specified manually. + refs: + type: array + description: 'List of resource references' + items: + type: string + description: A string reference like `` or ``. + dev: + type: object + description: Overrides any properties in development environment. + prod: + type: object + description: Overrides any properties in production environment. \ No newline at end of file diff --git a/runtime/parser/schema/explore.schema.yaml b/runtime/parser/schema/explore.schema.yaml new file mode 100644 index 00000000000..88ba481b319 --- /dev/null +++ b/runtime/parser/schema/explore.schema.yaml @@ -0,0 +1,285 @@ +$schema: 'http://json-schema.org/draft-07/schema#' +$id: explore.schema.yaml +title: Explore YAML +type: object +description: In your Rill project directory, create a explore dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. +allOf: + - title: Properties + type: object + properties: + type: + type: string + const: explore + description: Refers to the resource type and must be `explore` + display_name: + type: string + description: Refers to the display name for the explore dashboard + description: + type: string + description: Refers to the description of the explore dashboard + banner: + type: string + description: Refers to the custom banner displayed at the header of an explore dashboard + metrics_view: + type: string + description: Refers to the metrics view resource + dimensions: + description: List of dimension names. Use '*' to select all dimensions (default) + $ref: '#/definitions/explore/definitions/field_selector_properties' + measures: + description: List of measure names. Use ''*'' to select all measures (default) + $ref: '#/definitions/explore/definitions/field_selector_properties' + theme: + oneOf: + - type: string + description: Name of an existing theme to apply to the dashboard + - $ref: '#/definitions/theme/definitions/theme_properties' + description: Inline theme configuration. + description: Name of the theme to use. Only one of theme and embedded_theme can be set. + time_ranges: + type: array + description: Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' + items: + $ref: '#/definitions/explore_time_range_properties' + time_zones: + type: array + description: Refers to the time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) + items: + type: string + lock_time_zone: + type: boolean + description: When true, the dashboard will be locked to the first time provided in the time_zones list. When no time_zones are provided, the dashboard will be locked to UTC + allow_custom_time_range: + type: boolean + description: Defaults to true, when set to false it will hide the ability to set a custom time range for the user. + defaults: + type: object + description: defines the defaults YAML struct + properties: + dimensions: + description: Provides the default dimensions to load on viewing the dashboard + $ref: '#/definitions/explore/definitions/field_selector_properties' + measures: + description: Provides the default measures to load on viewing the dashboard + $ref: '#/definitions/explore/definitions/field_selector_properties' + time_range: + description: Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + type: string + comparison_mode: + description: 'Controls how to compare current data with historical or categorical baselines. Options: `none` (no comparison), `time` (compares with past based on default_time_range), `dimension` (compares based on comparison_dimension values)' + type: string + comparison_dimension: + description: 'for dimension mode, specify the comparison dimension by name' + type: string + additionalProperties: false + embeds: + type: object + description: Configuration options for embedded dashboard views + properties: + hide_pivot: + type: boolean + description: When true, hides the pivot table view in embedded mode + additionalProperties: false + security: + description: Security rules to apply for access to the explore dashboard + $ref: '#/definitions/security_policy_properties' + required: + - type + - $ref: '#/definitions/common_properties' +definitions: + explore_time_range_properties: + oneOf: + - type: string + description: a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection + - type: object + description: Object containing time range and comparison configuration + properties: + range: + type: string + description: a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection + comparison_offsets: + type: array + description: list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + items: + oneOf: + - type: string + description: Offset string only (range is inferred) + - type: object + description: Object containing offset and range configuration for time comparison + properties: + offset: + type: string + description: Time offset for comparison (e.g., 'P1D' for one day ago) + range: + type: string + description: Custom time range for comparison period + additionalProperties: false + required: + - range + additionalProperties: false + security_policy_properties: + type: object + description: Defines security rules and access control policies for resources + properties: + access: + oneOf: + - type: string + description: SQL expression that evaluates to a boolean to determine access + - type: boolean + description: Direct boolean value to allow or deny access + description: Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + row_filter: + type: string + description: SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause + include: + type: array + description: List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded + items: + type: object + properties: + if: + type: string + description: Expression to decide if the column should be included or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean + names: + anyOf: + - type: array + description: List of specific field names to include + items: + type: string + - type: string + description: Wildcard '*' to include all fields + enum: + - '*' + description: List of fields to include. Should match the name of one of the dashboard's dimensions or measures + required: + - if + - names + exclude: + type: array + description: List of dimension or measure names to exclude from the dashboard. If exclude is defined all other dimensions and measures are included + items: + type: object + properties: + if: + type: string + description: Expression to decide if the column should be excluded or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean + names: + anyOf: + - type: array + description: List of specific field names to exclude + items: + type: string + - type: string + description: Wildcard '*' to exclude all fields + enum: + - '*' + description: List of fields to exclude. Should match the name of one of the dashboard's dimensions or measures + required: + - if + - names + rules: + type: array + description: List of detailed security rules that can be used to define complex access control policies + items: + type: object + description: Individual security rule definition + properties: + type: + type: string + enum: + - access + - field_access + - row_filter + description: Type of security rule - access (overall access), field_access (field-level access), or row_filter (row-level filtering) + action: + type: string + enum: + - allow + - deny + description: Whether to allow or deny access for this rule + if: + type: string + description: Conditional expression that determines when this rule applies. Must be a valid SQL expression that evaluates to a boolean + names: + type: array + items: + type: string + description: List of field names this rule applies to (for field_access type rules) + all: + type: boolean + description: When true, applies the rule to all fields (for field_access type rules) + sql: + type: string + description: SQL expression for row filtering (for row_filter type rules) + required: + - type + common_properties: + type: object + title: "Common Properties" + properties: + name: + type: string + description: Name is usually inferred from the filename, but can be specified manually. + refs: + type: array + description: 'List of resource references' + items: + type: string + description: A string reference like `` or ``. + dev: + type: object + description: Overrides any properties in development environment. + prod: + type: object + description: Overrides any properties in production environment. + explore: + definitions: + field_selector_properties: + oneOf: + - type: string + const: '*' + description: Wildcard(*) selector that includes all available fields in the selection + - type: array + items: + type: string + description: Explicit list of fields to include in the selection + - type: object + description: 'Advanced matching using regex, DuckDB expression, or exclusion' + properties: + regex: + type: string + description: Select dimensions using a regular expression + expr: + type: string + description: DuckDB SQL expression to select fields based on custom logic + exclude: + type: object + description: Select all dimensions except those listed here + additionalProperties: false + oneOf: + - required: + - regex + - required: + - expr + - required: + - exclude + theme: + definitions: + theme_properties: + type: object + properties: + colors: + type: object + description: Used to override the dashboard colors. Either primary or secondary color must be provided. + properties: + primary: + type: string + description: Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). + secondary: + type: string + description: Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. + anyOf: + - required: + - primary + - required: + - secondary \ No newline at end of file diff --git a/runtime/parser/schema/metrics_view.schema.yaml b/runtime/parser/schema/metrics_view.schema.yaml new file mode 100644 index 00000000000..066351ea57a --- /dev/null +++ b/runtime/parser/schema/metrics_view.schema.yaml @@ -0,0 +1,324 @@ +$schema: 'http://json-schema.org/draft-07/schema#' +$id: metrics_view.schema.yaml +title: Metrics View YAML +type: object +description: In your Rill project directory, create a metrics view, `.yaml`, file in the `metrics` directory. Rill will ingest the metric view definition next time you run `rill start`. +allOf: + - title: Properties + type: object + properties: + type: + type: string + const: metrics_view + description: Refers to the resource type and must be `metrics_view` + display_name: + type: string + description: Refers to the display name for the metrics view + description: + type: string + description: Refers to the description for the metrics view + ai_instructions: + type: string + description: Extra instructions for AI agents. Used to guide natural language question answering and routing. + model: + type: string + description: Refers to the model powering the dashboard (either model or table is required) + database: + type: string + description: Refers to the database to use in the OLAP engine (to be used in conjunction with table). Otherwise, will use the default database or schema if not specified + database_schema: + type: string + description: Refers to the schema to use in the OLAP engine (to be used in conjunction with table). Otherwise, will use the default database or schema if not specified + table: + type: string + description: Refers to the table powering the dashboard, should be used instead of model for dashboards create from external OLAP tables (either table or model is required) + timeseries: + type: string + description: Refers to the timestamp column from your model that will underlie x-axis data in the line charts. If not specified, the line charts will not appear + watermark: + type: string + description: A SQL expression that tells us the max timestamp that the metrics are considered valid for. Usually does not need to be overwritten + smallest_time_grain: + type: string + description: 'Refers to the smallest time granularity the user is allowed to view. The valid values are: millisecond, second, minute, hour, day, week, month, quarter, year' + first_day_of_week: + type: integer + description: Refers to the first day of the week for time grain aggregation (for example, Sunday instead of Monday). The valid values are 1 through 7 where Monday=1 and Sunday=7 + first_month_of_year: + type: integer + description: Refers to the first month of the year for time grain aggregation. The valid values are 1 through 12 where January=1 and December=12 + dimensions: + type: array + description: Relates to exploring segments or dimensions of your data and filtering the dashboard + items: + type: object + properties: + name: + type: string + description: a stable identifier for the dimension + display_name: + type: string + description: a display name for your dimension + description: + type: string + description: a freeform text description of the dimension + column: + type: string + description: a categorical column + expression: + type: string + description: a non-aggregate expression such as string_split(domain, '.'). One of column and expression is required but cannot have both at the same time + unnest: + type: boolean + description: if true, allows multi-valued dimension to be unnested (such as lists) and filters will automatically switch to "contains" instead of exact match + uri: + type: + - string + - boolean + description: enable if your dimension is a clickable URL to enable single click navigation (boolean or valid SQL expression) + anyOf: + - required: + - column + - required: + - expression + measures: + type: array + description: Used to define the numeric aggregates of columns from your data model + items: + type: object + properties: + name: + type: string + description: a stable identifier for the measure + display_name: + type: string + description: the display name of your measure. + description: + type: string + description: a freeform text description of the dimension + type: + type: string + description: 'Measure calculation type: "simple" for basic aggregations, "derived" for calculations using other measures, or "time_comparison" for period-over-period analysis. Defaults to "simple" unless dependencies exist.' + expression: + type: string + description: a combination of operators and functions for aggregations + window: + description: A measure window can be defined as a keyword string (e.g. 'time' or 'all') or an object with detailed window configuration. + anyOf: + - type: string + enum: + - time + - 'true' + - all + description: 'Shorthand: `time` or `true` means time-partitioned, `all` means non-partitioned.' + - type: object + description: 'Detailed window configuration for measure calculations, allowing control over partitioning, ordering, and frame definition.' + properties: + partition: + type: boolean + description: 'Controls whether the window is partitioned. When true, calculations are performed within each partition separately.' + order: + $ref: '#/definitions/metrics_view/definitions/field_selectors_properties' + description: 'Specifies the fields to order the window by, determining the sequence of rows within each partition.' + frame: + type: string + description: 'Defines the window frame boundaries for calculations, specifying which rows are included in the window relative to the current row.' + additionalProperties: false + per: + $ref: '#/definitions/metrics_view/definitions/field_selectors_properties' + description: for per dimensions + requires: + $ref: '#/definitions/metrics_view/definitions/field_selectors_properties' + description: using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures + format_preset: + type: string + description: | + Controls the formatting of this measure using a predefined preset. Measures cannot have both `format_preset` and `format_d3`. If neither is supplied, the measure will be formatted using the `humanize` preset by default. + + Available options: + - `humanize`: Round numbers into thousands (K), millions(M), billions (B), etc. + - `none`: Raw output. + - `currency_usd`: Round to 2 decimal points with a dollar sign ($). + - `currency_eur`: Round to 2 decimal points with a euro sign (€). + - `percentage`: Convert a rate into a percentage with a % sign. + - `interval_ms`: Convert milliseconds into human-readable durations like hours (h), days (d), years (y), etc. (optional) + format_d3: + type: string + description: 'Controls the formatting of this measure using a [d3-format](https://d3js.org/d3-format) string. If an invalid format string is supplied, the measure will fall back to `format_preset: humanize`. A measure cannot have both `format_preset` and `format_d3`. If neither is provided, the humanize preset is used by default. Example: `format_d3: ".2f"` formats using fixed-point notation with two decimal places. Example: `format_d3: ",.2r"` formats using grouped thousands with two significant digits. (optional)' + format_d3_locale: + type: object + description: locale configuration passed through to D3, enabling changing the currency symbol among other things. For details, see the docs for D3's [formatLocale](https://d3js.org/d3-format#formatLocale) + additionalProperties: true + valid_percent_of_total: + type: boolean + description: a boolean indicating whether percent-of-total values should be rendered for this measure + treat_nulls_as: + type: string + description: used to configure what value to fill in for missing time buckets. This also works generally as COALESCING over non empty time buckets. + minItems: 1 + security: + $ref: '#/definitions/security_policy_properties' + description: Defines a security policy for the dashboard + required: + - type + - $ref: '#/definitions/common_properties' +definitions: + security_policy_properties: + type: object + description: Defines security rules and access control policies for resources + properties: + access: + oneOf: + - type: string + description: SQL expression that evaluates to a boolean to determine access + - type: boolean + description: Direct boolean value to allow or deny access + description: Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + row_filter: + type: string + description: SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause + include: + type: array + description: List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded + items: + type: object + properties: + if: + type: string + description: Expression to decide if the column should be included or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean + names: + anyOf: + - type: array + description: List of specific field names to include + items: + type: string + - type: string + description: Wildcard '*' to include all fields + enum: + - '*' + description: List of fields to include. Should match the name of one of the dashboard's dimensions or measures + required: + - if + - names + exclude: + type: array + description: List of dimension or measure names to exclude from the dashboard. If exclude is defined all other dimensions and measures are included + items: + type: object + properties: + if: + type: string + description: Expression to decide if the column should be excluded or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean + names: + anyOf: + - type: array + description: List of specific field names to exclude + items: + type: string + - type: string + description: Wildcard '*' to exclude all fields + enum: + - '*' + description: List of fields to exclude. Should match the name of one of the dashboard's dimensions or measures + required: + - if + - names + rules: + type: array + description: List of detailed security rules that can be used to define complex access control policies + items: + type: object + description: Individual security rule definition + properties: + type: + type: string + enum: + - access + - field_access + - row_filter + description: Type of security rule - access (overall access), field_access (field-level access), or row_filter (row-level filtering) + action: + type: string + enum: + - allow + - deny + description: Whether to allow or deny access for this rule + if: + type: string + description: Conditional expression that determines when this rule applies. Must be a valid SQL expression that evaluates to a boolean + names: + type: array + items: + type: string + description: List of field names this rule applies to (for field_access type rules) + all: + type: boolean + description: When true, applies the rule to all fields (for field_access type rules) + sql: + type: string + description: SQL expression for row filtering (for row_filter type rules) + required: + - type + common_properties: + type: object + title: "Common Properties" + properties: + name: + type: string + description: Name is usually inferred from the filename, but can be specified manually. + refs: + type: array + description: 'List of resource references' + items: + type: string + description: A string reference like `` or ``. + dev: + type: object + description: Overrides any properties in development environment. + prod: + type: object + description: Overrides any properties in production environment. + metrics_view: + definitions: + field_selectors_properties: + anyOf: + - type: string + description: 'Simple field name as a string.' + - type: array + description: 'List of field selectors, each can be a string or an object with detailed configuration.' + items: + anyOf: + - type: string + description: 'Shorthand field selector, interpreted as the name.' + - type: object + description: 'Detailed field selector configuration with name and optional time grain.' + properties: + name: + type: string + description: 'Name of the field to select.' + time_grain: + type: string + description: 'Time grain for time-based dimensions.' + enum: + - '' + - ms + - millisecond + - s + - second + - min + - minute + - h + - hour + - d + - day + - w + - week + - month + - q + - quarter + - 'y' + - year + required: + - name + additionalProperties: false + minItems: 1 \ No newline at end of file diff --git a/runtime/parser/schema/project.schema.yaml b/runtime/parser/schema/project.schema.yaml index a34ecd1deaa..b32effba1ba 100644 --- a/runtime/parser/schema/project.schema.yaml +++ b/runtime/parser/schema/project.schema.yaml @@ -21,1737 +21,16 @@ description: | ::: oneOf: - - $ref: '#/definitions/alert' - - $ref: '#/definitions/api' - - $ref: '#/definitions/canvas' - - $ref: '#/definitions/component' - - $ref: '#/definitions/connector' - - $ref: '#/definitions/explore' - - $ref: '#/definitions/metrics_view' - - $ref: '#/definitions/model' - - $ref: '#/definitions/theme' + - $ref: 'alert.schema.yaml#' + - $ref: 'api.schema.yaml#' + - $ref: 'canvas.schema.yaml#' + # - $ref: 'component.schema.yaml#' + - $ref: 'connector.schema.yaml#' + - $ref: 'explore.schema.yaml#' + - $ref: 'metrics_view.schema.yaml#' + - $ref: 'advanced-model.schema.yaml#' + - $ref: 'theme.schema.yaml#' definitions: - alert: - type: object - title: Alert YAML - description: Along with alertings at the dashboard level and can be created via the UI, there might be more extensive alerting that you might want to develop and can be done so the an alert.yaml. When creating an alert via a YAML file, you'll see this denoted in the UI as `Created through code`. - examples: - - # Example: To send alert when data lags by more than 1 day to slack channel #rill-cloud-alerts - type: alert - display_name: Data lags by more than 1 day - # Check the alert every hour. - refresh: - cron: 0 * * * * - # Query that returns non-empty results if the metrics lag by more than 1 day. - data: - sql: |- - SELECT * - FROM - ( - SELECT MAX(event_time) AS max_time - FROM rill_metrics_model - ) - WHERE max_time < NOW() - INTERVAL '1 day' - # Send notifications in Slack. - notify: - slack: - channels: - - '#rill-cloud-alerts' - allOf: - - title: Properties - type: object - properties: - type: - type: string - const: alert - description: Refers to the resource type and must be `alert` - display_name: - type: string - description: Refers to the display name for the alert - refresh: - $ref: '#/definitions/schedule_properties' - description: Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying data - intervals: - type: object - description: define the interval of the alert to check - properties: - duration: - type: string - description: a valid ISO8601 duration to define the interval duration - limit: - type: integer - description: maximum number of intervals to check for on invocation - minimum: 0 - check_unclosed: - type: boolean - description: 'boolean, whether unclosed intervals should be checked' - watermark: - type: string - enum: - - trigger_time - - inherit - description: Specifies how the watermark is determined for incremental processing. Use 'trigger_time' to set it at runtime or 'inherit' to use the upstream model's watermark. - timeout: - type: string - description: define the timeout of the alert in seconds (optional). - data: - description: Specifies one of the options to retrieve or compute the data used by alert - $ref: '#/definitions/data_properties' - for: - description: "Specifies how user identity or attributes should be evaluated for security policy enforcement." - oneOf: - - type: object - description: Specifies a unique user identifier for applying security policies. - properties: - user_id: - type: string - description: "The unique user ID used to evaluate security policies." - required: - - user_id - additionalProperties: false - - type: object - description: Specifies a user's email address for applying security policies. - properties: - user_email: - type: string - description: "The user's email address used to evaluate security policies." - format: email - required: - - user_email - additionalProperties: false - - type: object - description: Specifies a set of arbitrary user attributes for applying security policies. - properties: - attributes: - type: object - description: A dictionary of user attributes used to evaluate security policies. - additionalProperties: true - required: - - attributes - additionalProperties: false - on_recover: - type: boolean - description: Send an alert when a previously failing alert recovers. Defaults to false. - on_fail: - type: boolean - description: Send an alert when a failure occurs. Defaults to true. - on_error: - type: boolean - description: Send an alert when an error occurs during evaluation. Defaults to false. - renotify: - type: boolean - description: Enable repeated notifications for unresolved alerts. Defaults to false. - renotify_after: - type: string - description: Defines the re-notification interval for the alert (e.g., '10m','24h'), equivalent to snooze duration in UI, defaults to 'Off' - notify: - $ref: '#/definitions/notify_properties' - description: Defines how and where to send notifications. At least one method (email or Slack) is required. - annotations: - type: object - description: Key value pair used for annotations - additionalProperties: - type: string - required: - - type - - refresh - - data - - notify - - $ref: '#/definitions/common_properties' - api: - type: object - title: API YAML - description: In your Rill project directory, create a new file name `.yaml` in the `apis` directory containing a custom API definition. See comprehensive documentation on how to define and use [custom APIs](/integrate/custom-apis/index.md) - examples: - - # Example: This api returns the top 10 authors by net line changes since the specified date provided in the arguments. - type: api - name: metrics_view_api - metrics_sql: |- - SELECT author_name, net_line_changes - FROM advanced_metrics_view - where author_date > '{{ .args.date }}' - order by net_line_changes DESC - limit 10 - allOf: - - title: Properties - type: object - properties: - type: - type: string - const: api - description: Refers to the resource type and must be `api` - openapi: - type: object - description: OpenAPI specification for the API endpoint - properties: - summary: - type: string - description: A brief description of what the API endpoint does - parameters: - type: array - description: List of parameters that the API endpoint accepts - items: - type: object - additionalProperties: true - request_schema: - type: object - description: JSON schema for the request body (use nested YAML instead of a JSON string) - additionalProperties: true - response_schema: - type: object - description: JSON schema for the response body (use nested YAML instead of a JSON string) - additionalProperties: true - security: - $ref: '#/definitions/security_policy_properties' - skip_nested_security: - type: boolean - description: Flag to control security inheritance - allOf: - - $ref: '#/definitions/data_properties' - required: - - type - canvas: - type: object - title: Canvas YAML - description: In your Rill project directory, create a explore dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. - allOf: - - title: Properties - type: object - properties: - type: - type: string - const: canvas - description: Refers to the resource type and must be `canvas` - display_name: - type: string - description: Refers to the display name for the canvas - banner: - type: string - description: Refers to the custom banner displayed at the header of an Canvas dashboard - max_width: - type: integer - description: Max width in pixels of the canvas - minimum: 0 - gap_x: - type: integer - description: Horizontal gap in pixels of the canvas - minimum: 0 - gap_y: - type: integer - description: Vertical gap in pixels of the canvas - minimum: 0 - theme: - oneOf: - - type: string - description: Name of an existing theme to apply to the dashboard - - $ref: '#/definitions/theme/definitions/theme_properties' - description: Inline theme configuration. - description: Name of the theme to use. Only one of theme and embedded_theme can be set. - allow_custom_time_range: - type: boolean - description: Defaults to true, when set to false it will hide the ability to set a custom time range for the user. - time_ranges: - type: array - description: Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' - items: - $ref: '#/definitions/explore_time_range_properties' - time_zones: - type: array - description: Refers to the time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) - items: - type: string - filters: - type: object - description: Indicates if filters should be enabled for the canvas. - properties: - enable: - type: boolean - description: Toggles filtering functionality for the canvas dashboard. - additionalProperties: false - defaults: - type: object - description: Preset UI state to show by default - properties: - time_range: - type: string - description: Default time range to display when the dashboard loads - comparison_mode: - type: string - description: Default comparison mode for metrics (none, time, or dimension) - comparison_dimension: - type: string - description: Default dimension to use for comparison when comparison_mode is 'dimension' - additionalProperties: false - variables: - type: array - description: Variables that can be used in the canvas - items: - $ref: '#/definitions/component_variable_properties' - rows: - type: array - description: Refers to all of the rows displayed on the Canvas - items: - type: object - properties: - height: - type: string - description: Height of the row in px - items: - type: array - description: List of components to display in the row - items: - type: object - properties: - component: - type: string - description: Name of the component to display - width: - type: - - string - - integer - description: Width of the component (can be a number or string with unit) - additionalProperties: true - additionalProperties: false - security: - $ref: '#/definitions/security_policy_properties' - description: Security rules to apply for access to the canvas - required: - - type - - rows - - $ref: '#/definitions/common_properties' - component: - type: object - title: Component YAML - description: Defines a reusable dashboard component that can be embedded in canvas dashboards - allOf: - - title: Properties - type: object - properties: - type: - type: string - const: component - description: Refers to the resource type and must be `component` - display_name: - type: string - description: Refers to the display name for the component - description: - type: string - description: Detailed description of the component's purpose and functionality - input: - type: array - description: List of input variables that can be passed to the component - items: - $ref: '#/definitions/component_variable_properties' - output: - description: Output variable that the component produces - $ref: '#/definitions/component_variable_properties' - required: - - type - - $ref: '#/definitions/common_properties' - connector: - type: object - title: Connector YAML - description: | - Connector YAML files define how Rill connects to external data sources and OLAP engines. Each connector specifies a driver type and its required connection parameters. - allOf: - - title: Properties - type: object - properties: - type: - type: string - const: connector - description: Refers to the resource type and must be `connector` - driver: - type: string - description: The type of connector, see [available connectors](#available-connector-types) (required) - required: - - type - - driver - # - $ref: '#/definitions/common_properties' - - oneOf: - - $ref: '#/definitions/connector/definitions/athena' - - $ref: '#/definitions/connector/definitions/azure' - - $ref: '#/definitions/connector/definitions/bigquery' - - $ref: '#/definitions/connector/definitions/clickhouse' - - $ref: '#/definitions/connector/definitions/druid' - - $ref: '#/definitions/connector/definitions/duckdb' - - $ref: '#/definitions/connector/definitions/gcs' - - $ref: '#/definitions/connector/definitions/https' - - $ref: '#/definitions/connector/definitions/local_file' - - $ref: '#/definitions/connector/definitions/motherduck' - - $ref: '#/definitions/connector/definitions/mysql' - - $ref: '#/definitions/connector/definitions/pinot' - - $ref: '#/definitions/connector/definitions/postgres' - - $ref: '#/definitions/connector/definitions/redshift' - - $ref: '#/definitions/connector/definitions/s3' - - $ref: '#/definitions/connector/definitions/salesforce' - - $ref: '#/definitions/connector/definitions/slack' - - $ref: '#/definitions/connector/definitions/snowflake' - - $ref: '#/definitions/connector/definitions/sqlite' - definitions: - athena: - type: object - title: athena - description: Configuration properties specific to the athena - properties: - driver: - type: string - description: Refers to the driver type and must be driver `athena` - const: athena - aws_access_key_id: - type: string - description: AWS Access Key ID used for authentication. Required when using static credentials directly or as base credentials for assuming a role. - aws_secret_access_key: - type: string - description: AWS Secret Access Key paired with the Access Key ID. Required when using static credentials directly or as base credentials for assuming a role. - aws_access_token: - type: string - description: AWS session token used with temporary credentials. Required only if the Access Key and Secret Key are part of a temporary session credentials. - role_arn: - type: string - description: ARN of the IAM role to assume. When specified, the SDK uses the base credentials to call STS AssumeRole and obtain temporary credentials scoped to this role. - role_session_name: - type: string - description: Session name to associate with the STS AssumeRole session. Used only if 'role_arn' is specified. Useful for identifying and auditing the session. - external_id: - type: string - description: External ID required by some roles when assuming them, typically for cross-account access. Used only if 'role_arn' is specified and the role's trust policy requires it. - workgroup: - type: string - description: Athena workgroup to use for query execution. Defaults to 'primary' if not specified. - output_location: - type: string - description: S3 URI where Athena query results should be stored (e.g., s3://your-bucket/athena/results/). Optional if the selected workgroup has a default result configuration. - aws_region: - type: string - description: AWS region where Athena and the result S3 bucket are located (e.g., us-east-1). Defaults to 'us-east-1' if not specified. - allow_host_access: - type: boolean - description: Allow the Athena client to access host environment configurations such as environment variables or local AWS credential files. Defaults to true, enabling use of credentials and settings from the host environment unless explicitly disabled. - required: - - driver - azure: - type: object - title: azure - description: Configuration properties specific to the azure - properties: - driver: - type: string - description: Refers to the driver type and must be driver `azure` - const: azure - azure_storage_account: - type: string - description: Azure storage account name - azure_storage_key: - type: string - description: Azure storage access key - azure_storage_sas_token: - type: string - description: Optional azure SAS token for authentication - azure_storage_connection_string: - type: string - description: Optional azure connection string for storage account - azure_storage_bucket: - type: string - description: Name of the Azure Blob Storage container (equivalent to an S3 bucket) - allow_host_access: - type: boolean - description: Allow access to host environment configuration - required: - - driver - - azure_storage_bucket - bigquery: - type: object - title: bigquery - description: Configuration properties specific to the bigquery - properties: - driver: - type: string - description: Refers to the driver type and must be driver `bigquery` - const: bigquery - google_application_credentials: - type: string - description: Raw contents of the Google Cloud service account key (in JSON format) used for authentication. - project_id: - type: string - description: ID of the Google Cloud project to use for BigQuery operations. This can be omitted only if the project ID is included in the service account key. - allow_host_access: - type: boolean - description: Enable the BigQuery client to use credentials from the host environment when no service account JSON is provided. This includes Application Default Credentials from environment variables, local credential files, or the Google Compute Engine metadata server. Defaults to true, allowing seamless authentication in GCP environments. - required: - - driver - clickhouse: - type: object - title: clickhouse - description: Configuration properties specific to the clickhouse - properties: - driver: - type: string - description: Refers to the driver type and must be driver `clickhouse` - const: clickhouse - managed: - type: boolean - description: '`true` means Rill will provision the connector using the default provisioner. `false` disables automatic provisioning.' - mode: - type: string - description: "`read` - Controls the operation mode for the ClickHouse connection. Defaults to 'read' for safe operation with external databases. Set to 'readwrite' to enable model creation and table mutations. Note: When 'managed: true', this is automatically set to 'readwrite'." - dsn: - type: string - description: DSN(Data Source Name) for the ClickHouse connection - username: - type: string - description: Username for authentication - password: - type: string - description: Password for authentication - host: - type: string - description: Host where the ClickHouse instance is running - port: - type: integer - description: Port where the ClickHouse instance is accessible - database: - type: string - description: Name of the ClickHouse database within the cluster - ssl: - type: boolean - description: Indicates whether a secured SSL connection is required - cluster: - type: string - description: 'Cluster name, required for running distributed queries' - log_queries: - type: boolean - description: Controls whether to log raw SQL queries - settings_override: - type: string - description: override the default settings used in queries. example `readonly = 1, session_timezone = 'UTC'` - embed_port: - type: integer - description: Port to run ClickHouse locally (0 for random port) - can_scale_to_zero: - type: boolean - description: Indicates if the database can scale to zero - max_open_conns: - type: integer - description: Maximum number of open connections to the database - max_idle_conns: - type: integer - description: Maximum number of idle connections in the pool - dial_timeout: - type: string - description: Timeout for dialing the ClickHouse server - conn_max_lifetime: - type: string - description: Maximum time a connection may be reused - read_timeout: - type: string - description: Maximum time for a connection to read data - required: - - driver - druid: - type: object - title: druid - description: Configuration properties specific to the druid - properties: - driver: - type: string - description: Refers to the driver type and must be driver `druid` - const: druid - dsn: - type: string - description: Data Source Name (DSN) for connecting to Druid - username: - type: string - description: Username for authenticating with Druid - password: - type: string - description: Password for authenticating with Druid - host: - type: string - description: Hostname of the Druid coordinator or broker - port: - type: integer - description: Port number of the Druid service - ssl: - type: boolean - description: Enable SSL for secure connection - log_queries: - type: boolean - description: Log raw SQL queries sent to Druid - max_open_conns: - type: integer - description: Maximum number of open database connections (0 = default, -1 = unlimited) - skip_version_check: - type: boolean - description: Skip checking Druid version compatibility - required: - - driver - - dsn - duckdb: - type: object - title: duckdb - description: Configuration properties specific to the duckdb - properties: - driver: - type: string - description: Refers to the driver type and must be driver `duckdb` - const: duckdb - pool_size: - type: integer - description: Number of concurrent connections and queries allowed - allow_host_access: - type: boolean - description: Whether access to the local environment and file system is allowed - cpu: - type: integer - description: Number of CPU cores available to the database - memory_limit_gb: - type: integer - description: Amount of memory in GB available to the database - read_write_ratio: - type: number - description: Ratio of resources allocated to the read database; used to divide CPU and memory - init_sql: - type: string - description: is executed during database initialization. - conn_init_sql: - type: string - description: is executed when a new connection is initialized. - secrets: - type: string - description: Comma-separated list of other connector names to create temporary secrets for in DuckDB before executing a model. - log_queries: - type: boolean - description: Whether to log raw SQL queries executed through OLAP - required: - - driver - gcs: - type: object - title: gcs - description: Configuration properties specific to the gcs - properties: - driver: - type: string - description: Refers to the driver type and must be driver `gcs` - const: gcs - google_application_credentials: - type: string - description: Google Cloud credentials JSON string - bucket: - type: string - description: Name of gcs bucket - allow_host_access: - type: boolean - description: Allow access to host environment configuration - key_id: - type: string - description: Optional S3-compatible Key ID when used in compatibility mode - secret: - type: string - description: Optional S3-compatible Secret when used in compatibility mode - required: - - driver - - bucket - https: - type: object - title: https - description: Configuration properties specific to the https - properties: - driver: - type: string - description: Refers to the driver type and must be driver `https` - const: https - path: - type: string - description: The full HTTPS URI to fetch data from - headers: - type: object - description: HTTP headers to include in the request - additionalProperties: - type: string - required: - - driver - - path - local_file: - type: object - title: local_file - description: Configuration properties specific to the local_file - properties: - driver: - type: string - description: Refers to the driver type and must be driver `local_file` - const: local_file - dsn: - type: string - description: Data Source Name (DSN) indicating the file path or location of the local file - allow_host_access: - type: boolean - description: Flag to indicate if access to host-level file paths is permitted - required: - - driver - - dsn - motherduck: - type: object - title: motherduck - description: Configuration properties specific to the motherduck - properties: - driver: - type: string - description: Refers to the driver type and must be driver `motherduck` - const: motherduck - dsn: - type: string - description: Data Source Name (DSN) specifying the MotherDuck connection endpoint - token: - type: string - description: Authentication token for accessing MotherDuck (secret) - required: - - driver - - dsn - - token - mysql: - type: object - title: mysql - description: Configuration properties specific to the mysql - properties: - driver: - type: string - description: Refers to the driver type and must be driver `mysql` - const: mysql - dsn: - type: string - description: DSN(Data Source Name) for the mysql connection - host: - type: string - description: Hostname of the MySQL server - port: - type: integer - description: Port number for the MySQL server - database: - type: string - description: Name of the MySQL database - user: - type: string - description: Username for authentication - password: - type: string - description: Password for authentication - ssl_mode: - type: string - description: SSL mode can be DISABLED, PREFERRED or REQUIRED - required: - - driver - pinot: - type: object - title: pinot - description: Configuration properties specific to the pinot - properties: - driver: - type: string - description: Refers to the driver type and must be driver `pinot` - const: pinot - dsn: - type: string - description: DSN(Data Source Name) for the Pinot connection - username: - type: string - description: Username for authenticating with Pinot - password: - type: string - description: Password for authenticating with Pinot - broker_host: - type: string - description: Hostname of the Pinot broker - broker_port: - type: integer - description: Port number for the Pinot broker - controller_host: - type: string - description: Hostname of the Pinot controller - controller_port: - type: integer - description: Port number for the Pinot controller - ssl: - type: boolean - description: Enable SSL connection to Pinot - log_queries: - type: boolean - description: Log raw SQL queries executed through Pinot - max_open_conns: - type: integer - description: Maximum number of open connections to the Pinot database - required: - - driver - - dsn - - broker_host - - controller_host - postgres: - type: object - title: postgres - description: Configuration properties specific to the postgres - properties: - driver: - type: string - description: Refers to the driver type and must be driver `postgres` - const: postgres - dsn: - type: string - description: DSN(Data Source Name) for the postgres connection - host: - type: string - description: Hostname of the Postgres server - port: - type: string - description: Port number for the Postgres server - dbname: - type: string - description: Name of the Postgres database - user: - type: string - description: Username for authentication - password: - type: string - description: Password for authentication - sslmode: - type: string - description: SSL mode can be disable, allow, prefer or require - required: - - driver - redshift: - type: object - title: redshift - description: Configuration properties specific to the redshift - properties: - driver: - type: string - description: Refers to the driver type and must be driver `redshift` - const: redshift - aws_access_key_id: - type: string - description: AWS Access Key ID used for authenticating with Redshift. - aws_secret_access_key: - type: string - description: AWS Secret Access Key used for authenticating with Redshift. - aws_access_token: - type: string - description: AWS Session Token for temporary credentials (optional). - region: - type: string - description: AWS region where the Redshift cluster or workgroup is hosted (e.g., 'us-east-1'). - database: - type: string - description: Name of the Redshift database to query. - workgroup: - type: string - description: Workgroup name for Redshift Serverless, in case of provisioned Redshift clusters use 'cluster_identifier'. - cluster_identifier: - type: string - description: Cluster identifier for provisioned Redshift clusters, in case of Redshift Serverless use 'workgroup' . - required: - - driver - - aws_access_key_id - - aws_secret_access_key - - database - s3: - type: object - title: s3 - description: Configuration properties specific to the s3 - properties: - driver: - type: string - description: Refers to the driver type and must be driver `s3` - const: s3 - aws_access_key_id: - type: string - description: AWS Access Key ID used for authentication - aws_secret_access_key: - type: string - description: AWS Secret Access Key used for authentication - aws_access_token: - type: string - description: Optional AWS session token for temporary credentials - bucket: - type: string - description: Name of s3 bucket - endpoint: - type: string - description: Optional custom endpoint URL for S3-compatible storage - region: - type: string - description: AWS region of the S3 bucket - allow_host_access: - type: boolean - description: Allow access to host environment configuration - retain_files: - type: boolean - description: Whether to retain intermediate files after processing - required: - - driver - - bucket - salesforce: - type: object - title: salesforce - description: Configuration properties specific to the salesforce - properties: - driver: - type: string - description: Refers to the driver type and must be driver `salesforce` - const: salesforce - username: - type: string - description: Salesforce account username - password: - type: string - description: Salesforce account password (secret) - key: - type: string - description: Authentication key for Salesforce (secret) - endpoint: - type: string - description: Salesforce API endpoint URL - client_id: - type: string - description: Client ID used for Salesforce OAuth authentication - required: - - driver - - username - - endpoint - slack: - type: object - title: slack - description: Configuration properties specific to the slack - properties: - driver: - type: string - description: Refers to the driver type and must be driver `slack` - const: slack - bot_token: - type: string - description: Bot token used for authenticating Slack API requests - required: - - driver - - bot_token - snowflake: - type: object - title: snowflake - description: Configuration properties specific to the snowflake - properties: - driver: - type: string - description: Refers to the driver type and must be driver `snowflake` - const: snowflake - dsn: - type: string - description: DSN (Data Source Name) for the Snowflake connection - parallel_fetch_limit: - type: integer - description: Maximum number of concurrent fetches during query execution - required: - - driver - - dsn - sqlite: - type: object - title: sqlite - description: Configuration properties specific to the sqlite - properties: - driver: - type: string - description: Refers to the driver type and must be driver `sqlite` - const: sqlite - dsn: - type: string - description: DSN(Data Source Name) for the sqlite connection - required: - - driver - - dsn - explore: - type: object - title: Explore YAML - description: In your Rill project directory, create a explore dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. - allOf: - - title: Properties - type: object - properties: - type: - type: string - const: explore - description: Refers to the resource type and must be `explore` - display_name: - type: string - description: Refers to the display name for the explore dashboard - description: - type: string - description: Refers to the description of the explore dashboard - banner: - type: string - description: Refers to the custom banner displayed at the header of an explore dashboard - metrics_view: - type: string - description: Refers to the metrics view resource - dimensions: - description: List of dimension names. Use '*' to select all dimensions (default) - $ref: '#/definitions/explore/definitions/field_selector_properties' - measures: - description: List of measure names. Use ''*'' to select all measures (default) - $ref: '#/definitions/explore/definitions/field_selector_properties' - theme: - oneOf: - - type: string - description: Name of an existing theme to apply to the dashboard - - $ref: '#/definitions/theme/definitions/theme_properties' - description: Inline theme configuration. - description: Name of the theme to use. Only one of theme and embedded_theme can be set. - time_ranges: - type: array - description: Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' - items: - $ref: '#/definitions/explore_time_range_properties' - time_zones: - type: array - description: Refers to the time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) - items: - type: string - lock_time_zone: - type: boolean - description: When true, the dashboard will be locked to the first time provided in the time_zones list. When no time_zones are provided, the dashboard will be locked to UTC - allow_custom_time_range: - type: boolean - description: Defaults to true, when set to false it will hide the ability to set a custom time range for the user. - defaults: - type: object - description: defines the defaults YAML struct - properties: - dimensions: - description: Provides the default dimensions to load on viewing the dashboard - $ref: '#/definitions/explore/definitions/field_selector_properties' - measures: - description: Provides the default measures to load on viewing the dashboard - $ref: '#/definitions/explore/definitions/field_selector_properties' - time_range: - description: Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) - type: string - comparison_mode: - description: 'Controls how to compare current data with historical or categorical baselines. Options: `none` (no comparison), `time` (compares with past based on default_time_range), `dimension` (compares based on comparison_dimension values)' - type: string - comparison_dimension: - description: 'for dimension mode, specify the comparison dimension by name' - type: string - additionalProperties: false - embeds: - type: object - description: Configuration options for embedded dashboard views - properties: - hide_pivot: - type: boolean - description: When true, hides the pivot table view in embedded mode - additionalProperties: false - security: - description: Security rules to apply for access to the explore dashboard - $ref: '#/definitions/security_policy_properties' - required: - - type - - $ref: '#/definitions/common_properties' - definitions: - field_selector_properties: - oneOf: - - type: string - const: '*' - description: Wildcard(*) selector that includes all available fields in the selection - - type: array - items: - type: string - description: Explicit list of fields to include in the selection - - type: object - description: 'Advanced matching using regex, DuckDB expression, or exclusion' - properties: - regex: - type: string - description: Select dimensions using a regular expression - expr: - type: string - description: DuckDB SQL expression to select fields based on custom logic - exclude: - type: object - description: Select all dimensions except those listed here - additionalProperties: false - oneOf: - - required: - - regex - - required: - - expr - - required: - - exclude - metrics_view: - type: object - title: Metrics View YAML - description: In your Rill project directory, create a metrics view, `.yaml`, file in the `metrics` directory. Rill will ingest the metric view definition next time you run `rill start`. - allOf: - - title: Properties - type: object - properties: - type: - type: string - const: metrics_view - description: Refers to the resource type and must be `metrics_view` - display_name: - type: string - description: Refers to the display name for the metrics view - description: - type: string - description: Refers to the description for the metrics view - ai_instructions: - type: string - description: Extra instructions for AI agents. Used to guide natural language question answering and routing. - model: - type: string - description: Refers to the model powering the dashboard (either model or table is required) - database: - type: string - description: Refers to the database to use in the OLAP engine (to be used in conjunction with table). Otherwise, will use the default database or schema if not specified - database_schema: - type: string - description: Refers to the schema to use in the OLAP engine (to be used in conjunction with table). Otherwise, will use the default database or schema if not specified - table: - type: string - description: Refers to the table powering the dashboard, should be used instead of model for dashboards create from external OLAP tables (either table or model is required) - timeseries: - type: string - description: Refers to the timestamp column from your model that will underlie x-axis data in the line charts. If not specified, the line charts will not appear - watermark: - type: string - description: A SQL expression that tells us the max timestamp that the metrics are considered valid for. Usually does not need to be overwritten - smallest_time_grain: - type: string - description: 'Refers to the smallest time granularity the user is allowed to view. The valid values are: millisecond, second, minute, hour, day, week, month, quarter, year' - first_day_of_week: - type: integer - description: Refers to the first day of the week for time grain aggregation (for example, Sunday instead of Monday). The valid values are 1 through 7 where Monday=1 and Sunday=7 - first_month_of_year: - type: integer - description: Refers to the first month of the year for time grain aggregation. The valid values are 1 through 12 where January=1 and December=12 - dimensions: - type: array - description: Relates to exploring segments or dimensions of your data and filtering the dashboard - items: - type: object - properties: - name: - type: string - description: a stable identifier for the dimension - display_name: - type: string - description: a display name for your dimension - description: - type: string - description: a freeform text description of the dimension - column: - type: string - description: a categorical column - expression: - type: string - description: a non-aggregate expression such as string_split(domain, '.'). One of column and expression is required but cannot have both at the same time - unnest: - type: boolean - description: if true, allows multi-valued dimension to be unnested (such as lists) and filters will automatically switch to "contains" instead of exact match - uri: - type: - - string - - boolean - description: enable if your dimension is a clickable URL to enable single click navigation (boolean or valid SQL expression) - anyOf: - - required: - - column - - required: - - expression - measures: - type: array - description: Used to define the numeric aggregates of columns from your data model - items: - type: object - properties: - name: - type: string - description: a stable identifier for the measure - display_name: - type: string - description: the display name of your measure. - description: - type: string - description: a freeform text description of the dimension - type: - type: string - description: 'Measure calculation type: "simple" for basic aggregations, "derived" for calculations using other measures, or "time_comparison" for period-over-period analysis. Defaults to "simple" unless dependencies exist.' - expression: - type: string - description: a combination of operators and functions for aggregations - window: - description: A measure window can be defined as a keyword string (e.g. 'time' or 'all') or an object with detailed window configuration. - anyOf: - - type: string - enum: - - time - - 'true' - - all - description: 'Shorthand: `time` or `true` means time-partitioned, `all` means non-partitioned.' - - type: object - description: 'Detailed window configuration for measure calculations, allowing control over partitioning, ordering, and frame definition.' - properties: - partition: - type: boolean - description: 'Controls whether the window is partitioned. When true, calculations are performed within each partition separately.' - order: - $ref: '#/definitions/metrics_view/definitions/field_selectors_properties' - description: 'Specifies the fields to order the window by, determining the sequence of rows within each partition.' - frame: - type: string - description: 'Defines the window frame boundaries for calculations, specifying which rows are included in the window relative to the current row.' - additionalProperties: false - per: - $ref: '#/definitions/metrics_view/definitions/field_selectors_properties' - description: for per dimensions - requires: - $ref: '#/definitions/metrics_view/definitions/field_selectors_properties' - description: using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures - format_preset: - type: string - description: | - Controls the formatting of this measure using a predefined preset. Measures cannot have both `format_preset` and `format_d3`. If neither is supplied, the measure will be formatted using the `humanize` preset by default. - - Available options: - - `humanize`: Round numbers into thousands (K), millions(M), billions (B), etc. - - `none`: Raw output. - - `currency_usd`: Round to 2 decimal points with a dollar sign ($). - - `currency_eur`: Round to 2 decimal points with a euro sign (€). - - `percentage`: Convert a rate into a percentage with a % sign. - - `interval_ms`: Convert milliseconds into human-readable durations like hours (h), days (d), years (y), etc. (optional) - format_d3: - type: string - description: 'Controls the formatting of this measure using a [d3-format](https://d3js.org/d3-format) string. If an invalid format string is supplied, the measure will fall back to `format_preset: humanize`. A measure cannot have both `format_preset` and `format_d3`. If neither is provided, the humanize preset is used by default. Example: `format_d3: ".2f"` formats using fixed-point notation with two decimal places. Example: `format_d3: ",.2r"` formats using grouped thousands with two significant digits. (optional)' - format_d3_locale: - type: object - description: locale configuration passed through to D3, enabling changing the currency symbol among other things. For details, see the docs for D3's [formatLocale](https://d3js.org/d3-format#formatLocale) - additionalProperties: true - valid_percent_of_total: - type: boolean - description: a boolean indicating whether percent-of-total values should be rendered for this measure - treat_nulls_as: - type: string - description: used to configure what value to fill in for missing time buckets. This also works generally as COALESCING over non empty time buckets. - minItems: 1 - security: - $ref: '#/definitions/security_policy_properties' - description: Defines a security policy for the dashboard - required: - - type - - $ref: '#/definitions/common_properties' - definitions: - field_selectors_properties: - anyOf: - - type: string - description: 'Simple field name as a string.' - - type: array - description: 'List of field selectors, each can be a string or an object with detailed configuration.' - items: - anyOf: - - type: string - description: 'Shorthand field selector, interpreted as the name.' - - type: object - description: 'Detailed field selector configuration with name and optional time grain.' - properties: - name: - type: string - description: 'Name of the field to select.' - time_grain: - type: string - description: 'Time grain for time-based dimensions.' - enum: - - '' - - ms - - millisecond - - s - - second - - min - - minute - - h - - hour - - d - - day - - w - - week - - month - - q - - quarter - - 'y' - - year - required: - - name - additionalProperties: false - minItems: 1 - model: - type: object - title: Model YAML - allOf: - - title: Properties - type: object - properties: - type: - type: string - const: model - description: Refers to the resource type and must be `model` - refresh: - $ref: '#/definitions/schedule_properties' - description: Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying model data - connector: - type: string - description: | - Refers to the connector type or [named connector](./connector.md#name) for the source. - sql: - type: string - description: Raw SQL query to run against source - timeout: - type: string - description: The maximum time to wait for model ingestion - incremental: - type: boolean - description: whether incremental modeling is required (optional) - change_mode: - type: string - enum: - - reset - - manual - - patch - description: Configure how changes to the model specifications are applied (optional). 'reset' will drop and recreate the model automatically, 'manual' will require a manual full or incremental refresh to apply changes, and 'patch' will switch to the new logic without re-processing historical data (only applies for incremental models). - state: - $ref: '#/definitions/data_properties' - description: Refers to the explicitly defined state of your model, cannot be used with partitions (optional) - partitions: - $ref: '#/definitions/data_properties' - description: Refers to the how your data is partitioned, cannot be used with state. (optional) - materialize: - type: boolean - description: models will be materialized in olap - partitions_watermark: - type: string - description: Refers to a customizable timestamp that can be set to check if an object has been updated (optional). - partitions_concurrency: - type: integer - description: Refers to the number of concurrent partitions that can be read at the same time (optional). - stage: - type: object - properties: - connector: - type: string - description: Refers to the connector type for the staging table - required: - - connector - description: in the case of staging models, where an input source does not support direct write to the output and a staging table is required - additionalProperties: true - output: - type: object - description: to define the properties of output - properties: - table: - type: string - description: Name of the output table. If not specified, the model name is used. - materialize: - type: boolean - description: Whether to materialize the model as a table or view - connector: - type: string - description: Refers to the connector type for the output table. Can be `clickhouse` or `duckdb` and their named connector - incremental_strategy: - type: string - enum: - - append - - merge - - partition_overwrite - description: Strategy to use for incremental updates. Can be 'append', 'merge' or 'partition_overwrite' - unique_key: - type: array - items: - type: string - description: List of columns that uniquely identify a row for merge strategy - partition_by: - type: string - description: Column or expression to partition the table by - allOf: - - if: - title: Additional properties for `output` when `connector` is `clickhouse` - properties: - connector: - const: clickhouse - required: - - connector - then: - properties: - type: - type: string - description: Type to materialize the model into. Can be 'TABLE', 'VIEW' or 'DICTIONARY' - enum: - - TABLE - - VIEW - - DICTIONARY - columns: - type: string - description: Column names and types. Can also include indexes. If unspecified, detected from the query. - engine_full: - type: string - description: Full engine definition in SQL format. Can include partition keys, order, TTL, etc. - engine: - type: string - description: Table engine to use. Default is MergeTree - order_by: - type: string - description: ORDER BY clause. - partition_by: - type: string - description: Partition BY clause. - primary_key: - type: string - description: PRIMARY KEY clause. - sample_by: - type: string - description: SAMPLE BY clause. - ttl: - type: string - description: TTL settings for the table or columns. - table_settings: - type: string - description: Table-specific settings. - query_settings: - type: string - description: Settings used in insert/create table as select queries. - distributed_settings: - type: string - description: Settings for distributed table. - distributed_sharding_key: - type: string - description: Sharding key for distributed table. - dictionary_source_user: - type: string - description: User for accessing the source dictionary table (used if type is DICTIONARY). - dictionary_source_password: - type: string - description: Password for the dictionary source user. - required: - - type - - sql - - $ref: '#/definitions/common_properties' - - type: object - allOf: - - if: - title: Additional properties when `connector` is `athena` or [named connector](./connector.md#name) for athena - properties: - connector: - const: athena - required: - - connector - then: - $ref: '#/definitions/model/definitions/athena' - - if: - title: Additional properties when `connector` is `azure` or [named connector](./connector.md#name) of azure - properties: - connector: - const: azure - required: - - connector - then: - $ref: '#/definitions/model/definitions/azure' - - if: - title: Additional properties when `connector` is `bigquery` or [named connector](./connector.md#name) of bigquery - properties: - connector: - const: bigquery - required: - - connector - then: - $ref: '#/definitions/model/definitions/bigquery' - - if: - title: Additional properties when `connector` is `duckdb` or [named connector](./connector.md#name) of duckdb - properties: - connector: - const: duckdb - required: - - connector - then: - $ref: '#/definitions/model/definitions/duckdb' - - if: - title: Additional properties when `connector` is `gcs` or [named connector](./connector.md#name) of gcs - properties: - connector: - const: gcs - required: - - connector - then: - $ref: '#/definitions/model/definitions/gcs' - - if: - title: Additional properties when `connector` is `local_file` or [named connector](./connector.md#name) of local_file - properties: - connector: - const: local_file - required: - - connector - then: - $ref: '#/definitions/model/definitions/local_file' - - if: - title: Additional properties when `connector` is `redshift` or [named connector](./connector.md#name) of redshift - properties: - connector: - const: redshift - required: - - connector - then: - $ref: '#/definitions/model/definitions/redshift' - - if: - title: Additional properties when `connector` is `s3` or [named connector](./connector.md#name) of s3 - properties: - connector: - const: s3 - required: - - connector - then: - $ref: '#/definitions/model/definitions/s3' - - if: - title: Additional properties when `connector` is `salesforce` or [named connector](./connector.md#name) of salesforce - properties: - connector: - const: salesforce - required: - - connector - then: - $ref: '#/definitions/model/definitions/salesforce' - definitions: - athena: - type: object - properties: - output_location: - type: string - description: Output location for query results in S3. - workgroup: - type: string - description: AWS Athena workgroup to use for queries. - region: - type: string - description: AWS region to connect to Athena and the output location. - azure: - type: object - properties: - path: - type: string - description: Path to the source - account: - type: string - description: Account identifier - uri: - type: string - description: Source URI - extract: - type: object - description: Arbitrary key-value pairs for extraction settings - additionalProperties: true - glob: - type: object - description: Settings related to glob file matching. - properties: - max_total_size: - type: integer - description: Maximum total size (in bytes) matched by glob - max_objects_matched: - type: integer - description: Maximum number of objects matched by glob - max_objects_listed: - type: integer - description: Maximum number of objects listed in glob - page_size: - type: integer - description: Page size for glob listing - batch_size: - type: string - description: 'Size of a batch (e.g., ''100MB'')' - bigquery: - type: object - properties: - project_id: - type: string - description: ID of the BigQuery project. - duckdb: - type: object - properties: - path: - type: string - description: Path to the data source. - format: - type: string - description: 'Format of the data source (e.g., csv, json, parquet).' - pre_exec: - type: string - description: 'refers to SQL queries to run before the main query, available for DuckDB-based models. _(optional)_. Ensure `pre_exec` queries are idempotent. Use `IF NOT EXISTS` statements when applicable.' - post_exec: - type: string - description: 'refers to a SQL query that is run after the main query, available for DuckDB-based models. _(optional)_. Ensure `post_exec` queries are idempotent. Use `IF EXISTS` statements when applicable.' - examples: - - pre_exec: ATTACH IF NOT EXISTS 'dbname=postgres host=localhost port=5432 user=postgres password=postgres' AS postgres_db (TYPE POSTGRES); - sql: SELECT * FROM postgres_query('postgres_db', 'SELECT * FROM USERS') - post_exec: DETACH DATABASE IF EXISTS postgres_db - gcs: - type: object - properties: - path: - type: string - description: Path to the source - uri: - type: string - description: Source URI - extract: - type: object - description: key-value pairs for extraction settings - additionalProperties: true - glob: - type: object - description: Settings related to glob file matching. - properties: - max_total_size: - type: integer - description: Maximum total size (in bytes) matched by glob - max_objects_matched: - type: integer - description: Maximum number of objects matched by glob - max_objects_listed: - type: integer - description: Maximum number of objects listed in glob - page_size: - type: integer - description: Page size for glob listing - batch_size: - type: string - description: 'Size of a batch (e.g., ''100MB'')' - local_file: - type: object - properties: - path: - type: string - description: Path to the data source. - format: - type: string - description: 'Format of the data source (e.g., csv, json, parquet).' - redshift: - type: object - properties: - output_location: - type: string - description: S3 location where query results are stored. - workgroup: - type: string - description: Redshift Serverless workgroup to use. - database: - type: string - description: Name of the Redshift database. - cluster_identifier: - type: string - description: Identifier of the Redshift cluster. - role_arn: - type: string - description: ARN of the IAM role to assume for Redshift access. - region: - type: string - description: AWS region of the Redshift deployment. - s3: - type: object - properties: - region: - type: string - description: AWS region - endpoint: - type: string - description: AWS Endpoint - path: - type: string - description: Path to the source - uri: - type: string - description: Source URI - extract: - type: object - description: key-value pairs for extraction settings - additionalProperties: true - glob: - type: object - description: Settings related to glob file matching. - properties: - max_total_size: - type: integer - description: Maximum total size (in bytes) matched by glob - max_objects_matched: - type: integer - description: Maximum number of objects matched by glob - max_objects_listed: - type: integer - description: Maximum number of objects listed in glob - page_size: - type: integer - description: Page size for glob listing - batch_size: - type: string - description: 'Size of a batch (e.g., ''100MB'')' - salesforce: - type: object - properties: - soql: - type: string - description: SOQL query to execute against the Salesforce instance. - sobject: - type: string - description: Salesforce object (e.g., Account, Contact) targeted by the query. - queryAll: - type: boolean - description: Whether to include deleted and archived records in the query (uses queryAll API). - - theme: - type: object - title: Theme YAML - description: | - In your Rill project directory, create a `.yaml` file in any directory containing `type: theme`. Rill will automatically ingest the theme next time you run `rill start` or deploy to Rill Cloud. - - To apply that theme to a dashboard, add `default_theme: ` to the yaml file for that dashboard. Alternatively, you can add this to the end of the URL in your browser: `?theme=` - examples: - - # Example: You can copy this directly into your .yaml file - type: theme - colors: - primary: plum - secondary: violet - allOf: - - title: Properties - type: object - properties: - type: - type: string - const: theme - description: Refers to the resource type and must be `theme` - required: - - type - - $ref: '#/definitions/theme/definitions/theme_properties' - required: - - colors - - $ref: '#/definitions/common_properties' - definitions: - theme_properties: - type: object - properties: - colors: - type: object - description: Used to override the dashboard colors. Either primary or secondary color must be provided. - properties: - primary: - type: string - description: Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). - secondary: - type: string - description: Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. - anyOf: - - required: - - primary - - required: - - secondary common_properties: type: object title: "Common Properties" diff --git a/runtime/parser/schema/theme.schema.yaml b/runtime/parser/schema/theme.schema.yaml new file mode 100644 index 00000000000..0151d4d201b --- /dev/null +++ b/runtime/parser/schema/theme.schema.yaml @@ -0,0 +1,68 @@ +$schema: 'http://json-schema.org/draft-07/schema#' +$id: theme.schema.yaml +title: Theme YAML +type: object +description: | + In your Rill project directory, create a `.yaml` file in any directory containing `type: theme`. Rill will automatically ingest the theme next time you run `rill start` or deploy to Rill Cloud. + + To apply that theme to a dashboard, add `default_theme: ` to the yaml file for that dashboard. Alternatively, you can add this to the end of the URL in your browser: `?theme=` +examples: + - # Example: You can copy this directly into your .yaml file + type: theme + colors: + primary: plum + secondary: violet +allOf: + - title: Properties + type: object + properties: + type: + type: string + const: theme + description: Refers to the resource type and must be `theme` + required: + - type + - $ref: '#/definitions/theme/definitions/theme_properties' + required: + - colors + - $ref: '#/definitions/common_properties' +definitions: + theme: + definitions: + theme_properties: + type: object + properties: + colors: + type: object + description: Used to override the dashboard colors. Either primary or secondary color must be provided. + properties: + primary: + type: string + description: Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). + secondary: + type: string + description: Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. + anyOf: + - required: + - primary + - required: + - secondary + common_properties: + type: object + title: "Common Properties" + properties: + name: + type: string + description: Name is usually inferred from the filename, but can be specified manually. + refs: + type: array + description: 'List of resource references' + items: + type: string + description: A string reference like `` or ``. + dev: + type: object + description: Overrides any properties in development environment. + prod: + type: object + description: Overrides any properties in production environment. \ No newline at end of file From b6b987f0ef86dce9f40de6af847c74f0f11214b8 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 4 Aug 2025 13:39:04 -0600 Subject: [PATCH 04/32] nit --- cli/cmd/docs/generate_project.go | 4 ++-- runtime/parser/schema/connector.schema.yaml | 22 ++------------------- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index 7d5a3560748..e7139849b7c 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -549,8 +549,8 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req required = "_(required)_" } - doc.WriteString(fmt.Sprintf("\n\n### `%s`\n\n", propName)) - doc.WriteString(fmt.Sprintf("**Type:** %s\n\n**Description:** %s\n\n%s", + doc.WriteString(fmt.Sprintf("\n\n#### `%s`\n\n", propName)) + doc.WriteString(fmt.Sprintf("%s - %s %s", getPrintableType(propValue), getPrintableDescription(propValue, indent, "(no description)"), required)) diff --git a/runtime/parser/schema/connector.schema.yaml b/runtime/parser/schema/connector.schema.yaml index e33442cc0d4..1475074f897 100644 --- a/runtime/parser/schema/connector.schema.yaml +++ b/runtime/parser/schema/connector.schema.yaml @@ -18,7 +18,7 @@ allOf: required: - type - driver - - $ref: '#/definitions/common_properties' + - oneOf: - $ref: '#/definitions/athena' - $ref: '#/definitions/azure' @@ -39,6 +39,7 @@ allOf: - $ref: '#/definitions/slack' - $ref: '#/definitions/snowflake' - $ref: '#/definitions/sqlite' + definitions: athena: type: object @@ -615,22 +616,3 @@ definitions: required: - driver - dsn - common_properties: - type: object - title: "Common Properties" - properties: - name: - type: string - description: Name is usually inferred from the filename, but can be specified manually. - refs: - type: array - description: 'List of resource references' - items: - type: string - description: A string reference like `` or ``. - dev: - type: object - description: Overrides any properties in development environment. - prod: - type: object - description: Overrides any properties in production environment. \ No newline at end of file From d7d4b11f038a284a60065692f16770976feb9642 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 4 Aug 2025 13:52:07 -0600 Subject: [PATCH 05/32] link fixes --- .../parser/schema/advanced-model.schema.yaml | 18 +++++++++--------- runtime/parser/schema/project.schema.yaml | 2 +- runtime/parser/schema/rillyaml.schema.yaml | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/runtime/parser/schema/advanced-model.schema.yaml b/runtime/parser/schema/advanced-model.schema.yaml index e04f3c33825..448800d2839 100644 --- a/runtime/parser/schema/advanced-model.schema.yaml +++ b/runtime/parser/schema/advanced-model.schema.yaml @@ -150,7 +150,7 @@ allOf: - type: object allOf: - if: - title: Additional properties when `connector` is `athena` or [named connector](./connector.md#name) for athena + title: Additional properties when `connector` is [`athena`](./connector#athena) properties: connector: const: athena @@ -159,7 +159,7 @@ allOf: then: $ref: '#/definitions/model/definitions/athena' - if: - title: Additional properties when `connector` is `azure` or [named connector](./connector.md#name) of azure + title: Additional properties when `connector` is [`azure`](./connector#azure) properties: connector: const: azure @@ -168,7 +168,7 @@ allOf: then: $ref: '#/definitions/model/definitions/azure' - if: - title: Additional properties when `connector` is `bigquery` or [named connector](./connector.md#name) of bigquery + title: Additional properties when `connector` is [`bigquery`](./connector#bigquery) properties: connector: const: bigquery @@ -177,7 +177,7 @@ allOf: then: $ref: '#/definitions/model/definitions/bigquery' - if: - title: Additional properties when `connector` is `duckdb` or [named connector](./connector.md#name) of duckdb + title: Additional properties when `connector` is [`duckdb`](./connector#duckdb) properties: connector: const: duckdb @@ -186,7 +186,7 @@ allOf: then: $ref: '#/definitions/model/definitions/duckdb' - if: - title: Additional properties when `connector` is `gcs` or [named connector](./connector.md#name) of gcs + title: Additional properties when `connector` is [`gcs`](./connector#gcs) properties: connector: const: gcs @@ -195,7 +195,7 @@ allOf: then: $ref: '#/definitions/model/definitions/gcs' - if: - title: Additional properties when `connector` is `local_file` or [named connector](./connector.md#name) of local_file + title: Additional properties when `connector` is [`local_file`](./connector#local_file) properties: connector: const: local_file @@ -204,7 +204,7 @@ allOf: then: $ref: '#/definitions/model/definitions/local_file' - if: - title: Additional properties when `connector` is `redshift` or [named connector](./connector.md#name) of redshift + title: Additional properties when `connector` is [`redshift`](./connector#redshift) properties: connector: const: redshift @@ -213,7 +213,7 @@ allOf: then: $ref: '#/definitions/model/definitions/redshift' - if: - title: Additional properties when `connector` is `s3` or [named connector](./connector.md#name) of s3 + title: Additional properties when `connector` is [`s3`](./connector#s3) properties: connector: const: s3 @@ -222,7 +222,7 @@ allOf: then: $ref: '#/definitions/model/definitions/s3' - if: - title: Additional properties when `connector` is `salesforce` or [named connector](./connector.md#name) of salesforce + title: Additional properties when `connector` is [`salesforce`](./connector#salesforce) properties: connector: const: salesforce diff --git a/runtime/parser/schema/project.schema.yaml b/runtime/parser/schema/project.schema.yaml index b32effba1ba..eca03218469 100644 --- a/runtime/parser/schema/project.schema.yaml +++ b/runtime/parser/schema/project.schema.yaml @@ -7,7 +7,7 @@ description: | :::info Working with resources outside their native folders - It is possible to define resources (such as [models](model.md), [metrics-views](metrics-view.md), [dashboards](explore.md), [custom APIs](api.md), or [themes](theme.md)) within any nested folder within your Rill project directory. However, for any YAML configuration file, it is then imperative that the `type` property is then appropriately defined within the underlying resource configuration or Rill will not able to resolve the resource type correctly! + It is possible to define resources (such as [models](advanced-model.md), [metrics-views](metrics-view.md), [dashboards](explore.md), [custom APIs](api.md), or [themes](theme.md)) within any nested folder within your Rill project directory. However, for any YAML configuration file, it is then imperative that the `type` property is then appropriately defined within the underlying resource configuration or Rill will not able to resolve the resource type correctly! ::: diff --git a/runtime/parser/schema/rillyaml.schema.yaml b/runtime/parser/schema/rillyaml.schema.yaml index 597457830ba..599a0d7f651 100644 --- a/runtime/parser/schema/rillyaml.schema.yaml +++ b/runtime/parser/schema/rillyaml.schema.yaml @@ -38,7 +38,7 @@ allOf: - title: Project-wide defaults type: object description: | - In `rill.yaml`, project-wide defaults can be specified for a resource type within a project. Unless otherwise specified, _individual resources will inherit any defaults_ that have been specified in `rill.yaml`. For available properties that can be configured, please refer to the YAML specification for each individual resource type - [model](model.md), [metrics_view](metrics-view.md), and [explore](explore.md) + In `rill.yaml`, project-wide defaults can be specified for a resource type within a project. Unless otherwise specified, _individual resources will inherit any defaults_ that have been specified in `rill.yaml`. For available properties that can be configured, please refer to the YAML specification for each individual resource type - [model]advanced-model.md), [metrics_view](metrics-view.md), and [explore](explore.md) :::note Use plurals when specifying project-wide defaults In your `rill.yaml`, the top level property for the resource type needs to be **plural**, such as `models`, `metrics_views` and `explores`. @@ -46,7 +46,7 @@ allOf: :::info Hierarchy of inheritance and property overrides As a general rule of thumb, properties that have been specified at a more _granular_ level will supercede or override higher level properties that have been inherited. Therefore, in order of inheritance, Rill will prioritize properties in the following order: - 1. Individual [models](model.md)/[metrics_views](metrics-view.md)/[explore](explore.md) object level properties (e.g. `model.yaml` or `explore.yaml`) + 1. Individual [models](advanced-model.md)/[metrics_views](metrics-view.md)/[explore](explore.md) object level properties (e.g. `model.yaml` or `explore.yaml`) 2. [Environment](/docs/build/models/environments.md) level properties (e.g. a specific property that have been set for `dev`) 3. [Project-wide defaults](#project-wide-defaults) for a specific property and resource type ::: From a3784a0ad60acdeec5d9b562885bbb7712beae17 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 4 Aug 2025 13:54:11 -0600 Subject: [PATCH 06/32] Update generate_project.go --- cli/cmd/docs/generate_project.go | 141 +++++++++++++++---------------- 1 file changed, 67 insertions(+), 74 deletions(-) diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index e7139849b7c..b125b8777f9 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -173,7 +173,7 @@ func resolveRefsYAML(node, root *yaml.Node) error { fileName := strings.TrimSuffix(valNode.Value, "#") // Remove quotes if present fileName = strings.Trim(fileName, "'\"") - + // Load the external schema file externalSchema, err := parseSchemaYAML("runtime/parser/schema/" + fileName) if err != nil { @@ -346,10 +346,10 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req currentLevel := level title := getScalarValue(node, "title") description := getPrintableDescription(node, indent, "") - + // Check if this is a connector type for special handling isConnector := title == "Connector YAML" - + if level == 0 { doc.WriteString("---\n") doc.WriteString("note: GENERATED. DO NOT EDIT.\n") @@ -360,7 +360,6 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req doc.WriteString(fmt.Sprintf("\n\n%s", description)) } - level++ // level zero is to print base level info and its only onetime for a page so increasing level } else if level == 1 { if title != "" { @@ -425,7 +424,7 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req } } } - + for i, item := range oneOf.Content { if hasType(item) || hasProperties(item) || hasCombinators(item) { title := getScalarValue(item, "title") @@ -434,8 +433,8 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req } else { doc.WriteString(fmt.Sprintf("\n\n#### Option %d", i+1)) } - doc.WriteString(fmt.Sprintf("\n\n**Type:** %s\n\n**Description:** %s", - getPrintableType(item), + doc.WriteString(fmt.Sprintf("\n\n**Type:** %s\n\n**Description:** %s", + getPrintableType(item), getPrintableDescription(item, indent, "(no description)"))) doc.WriteString(generateDoc(sidebarPosition, level, item, indent+" ", getRequiredMapFromNode(item))) } @@ -460,51 +459,54 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req if isConnector && level == 1 { doc.WriteString("\n\n## Available Connector Types\n\n") doc.WriteString("Choose from the following connector types based on your data source:\n\n") - + // Find the oneOf section within allOf for _, item := range allOf.Content { - if oneOf := getNodeForKey(item, "oneOf"); oneOf != nil && oneOf.Kind == yaml.SequenceNode { - doc.WriteString("### OLAP Engines\n\n") - doc.WriteString("- [**DuckDB**](#duckdb) - Embedded DuckDB engine (default)\n") - doc.WriteString("- [**ClickHouse**](#clickhouse) - ClickHouse analytical database\n") - doc.WriteString("- [**MotherDuck**](#motherduck) - MotherDuck cloud database\n") - doc.WriteString("- [**Druid**](#druid) - Apache Druid\n") - doc.WriteString("- [**Pinot**](#pinot) - Apache Pinot\n\n") - - doc.WriteString("### Data Warehouses\n\n") - doc.WriteString("- [**Snowflake**](#snowflake) - Snowflake data warehouse\n") - doc.WriteString("- [**BigQuery**](#bigquery) - Google BigQuery\n") - doc.WriteString("- [**Redshift**](#redshift) - Amazon Redshift\n") - doc.WriteString("- [**Athena**](#athena) - Amazon Athena\n\n") - - doc.WriteString("### Databases\n\n") - doc.WriteString("- [**PostgreSQL**](#postgres) - PostgreSQL databases\n") - doc.WriteString("- [**MySQL**](#mysql) - MySQL databases\n") - doc.WriteString("- [**SQLite**](#sqlite) - SQLite databases\n\n") - - doc.WriteString("### Cloud Storage\n\n") - doc.WriteString("- [**GCS**](#gcs) - Google Cloud Storage\n") - doc.WriteString("- [**S3**](#s3) - Amazon S3 storage\n") - doc.WriteString("- [**Azure**](#azure) - Azure Blob Storage\n\n") - - doc.WriteString("### Other\n\n") - doc.WriteString("- [**HTTPS**](#https) - Public files via HTTP/HTTPS\n") - doc.WriteString("- [**Salesforce**](#salesforce) - Salesforce data\n") - doc.WriteString("- [**Slack**](#slack) - Slack data\n") - doc.WriteString("- [**Local File**](#local_file) - Local file system\n\n") - - doc.WriteString("## Connector Details\n\n") - break + oneOf := getNodeForKey(item, "oneOf") + if oneOf == nil || oneOf.Kind != yaml.SequenceNode { + continue } + + doc.WriteString("### OLAP Engines\n\n") + doc.WriteString("- [**DuckDB**](#duckdb) - Embedded DuckDB engine (default)\n") + doc.WriteString("- [**ClickHouse**](#clickhouse) - ClickHouse analytical database\n") + doc.WriteString("- [**MotherDuck**](#motherduck) - MotherDuck cloud database\n") + doc.WriteString("- [**Druid**](#druid) - Apache Druid\n") + doc.WriteString("- [**Pinot**](#pinot) - Apache Pinot\n\n") + + doc.WriteString("### Data Warehouses\n\n") + doc.WriteString("- [**Snowflake**](#snowflake) - Snowflake data warehouse\n") + doc.WriteString("- [**BigQuery**](#bigquery) - Google BigQuery\n") + doc.WriteString("- [**Redshift**](#redshift) - Amazon Redshift\n") + doc.WriteString("- [**Athena**](#athena) - Amazon Athena\n\n") + + doc.WriteString("### Databases\n\n") + doc.WriteString("- [**PostgreSQL**](#postgres) - PostgreSQL databases\n") + doc.WriteString("- [**MySQL**](#mysql) - MySQL databases\n") + doc.WriteString("- [**SQLite**](#sqlite) - SQLite databases\n\n") + + doc.WriteString("### Cloud Storage\n\n") + doc.WriteString("- [**GCS**](#gcs) - Google Cloud Storage\n") + doc.WriteString("- [**S3**](#s3) - Amazon S3 storage\n") + doc.WriteString("- [**Azure**](#azure) - Azure Blob Storage\n\n") + + doc.WriteString("### Other\n\n") + doc.WriteString("- [**HTTPS**](#https) - Public files via HTTP/HTTPS\n") + doc.WriteString("- [**Salesforce**](#salesforce) - Salesforce data\n") + doc.WriteString("- [**Slack**](#slack) - Slack data\n") + doc.WriteString("- [**Local File**](#local_file) - Local file system\n\n") + + doc.WriteString("## Connector Details\n\n") + break } } - + for _, item := range allOf.Content { // Skip oneOf items for connectors since we handle them separately if isConnector && getNodeForKey(item, "oneOf") != nil { continue } - + if hasIf(item) { ifNode := getNodeForKey(item, "if") title := getScalarValue(ifNode, "title") @@ -525,7 +527,7 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req if definitions := getNodeForKey(node, "definitions"); definitions != nil && definitions.Kind == yaml.MappingNode && isConnector && level == 1 { for i := 0; i < len(definitions.Content); i += 2 { connectorDef := definitions.Content[i+1] - + title := getScalarValue(connectorDef, "title") if title != "" { doc.WriteString(fmt.Sprintf("\n\n## %s\n\n", title)) @@ -537,7 +539,7 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req // Add YAML example after description doc.WriteString(generateConnectorExample(title, connectorDef)) } - + // Generate the connector definition documentation (properties, etc.) but skip the header // We need to process properties manually to avoid duplicate headers if properties := getNodeForKey(connectorDef, "properties"); properties != nil && properties.Kind == yaml.MappingNode { @@ -548,11 +550,11 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req if requiredFields := getRequiredMapFromNode(connectorDef); requiredFields[propName] { required = "_(required)_" } - + doc.WriteString(fmt.Sprintf("\n\n#### `%s`\n\n", propName)) - doc.WriteString(fmt.Sprintf("%s - %s %s", - getPrintableType(propValue), - getPrintableDescription(propValue, indent, "(no description)"), + doc.WriteString(fmt.Sprintf("%s - %s %s", + getPrintableType(propValue), + getPrintableDescription(propValue, indent, "(no description)"), required)) } } @@ -590,24 +592,15 @@ func hasCombinators(node *yaml.Node) bool { return getNodeForKey(node, "anyOf") != nil || getNodeForKey(node, "oneOf") != nil || getNodeForKey(node, "allOf") != nil } -func contains(slice []string, item string) bool { - for _, s := range slice { - if s == item { - return true - } - } - return false -} - func generateConnectorExample(connectorType string, connectorDef *yaml.Node) string { if connectorDef == nil { return "" } - + var example strings.Builder example.WriteString("```yaml\n") example.WriteString("type: connector # Must be `connector` (required)\n") - + // Get the driver from the schema and add it first driverAdded := false if driver := getNodeForKey(connectorDef, "driver"); driver != nil { @@ -616,52 +609,52 @@ func generateConnectorExample(connectorType string, connectorDef *yaml.Node) str driverAdded = true } } - + // Fallback: if driver wasn't found, use the connector type name if !driverAdded { example.WriteString(fmt.Sprintf("driver: %s # Must be `%s` _(required)_\n\n", strings.ToLower(connectorType), strings.ToLower(connectorType))) } - + // Get all properties from the schema if properties := getNodeForKey(connectorDef, "properties"); properties != nil && properties.Kind == yaml.MappingNode { for i := 0; i < len(properties.Content); i += 2 { propName := properties.Content[i].Value propValue := properties.Content[i+1] - + // Skip the driver property since we already added it if propName == "driver" { continue } - + // Get property description description := getPrintableDescription(propValue, "", "") if description == "" { description = "Property description" } - + // Get property type and generate appropriate example value propType := getScalarValue(propValue, "type") exampleValue := generateExampleValue(propName, propType, propValue) - + // Check if it's required required := "" if requiredFields := getRequiredMapFromNode(connectorDef); requiredFields[propName] { required = " _(required)_" } - + // Format the line with proper alignment example.WriteString(fmt.Sprintf("%s: %s", propName, exampleValue)) - + // Add padding for alignment padding := 35 - len(propName) - len(exampleValue) if padding > 0 { example.WriteString(strings.Repeat(" ", padding)) } - + example.WriteString(fmt.Sprintf("# %s%s\n", description, required)) } } - + example.WriteString("```\n\n") return example.String() } @@ -669,14 +662,14 @@ func generateConnectorExample(connectorType string, connectorDef *yaml.Node) str func generateExampleValue(propName, propType string, propNode *yaml.Node) string { // Check for const values first if constVal := getScalarValue(propNode, "const"); constVal != "" { - return fmt.Sprintf("\"%s\"", constVal) + return fmt.Sprintf("%q", constVal) } - + // Check for enum values if enum := getNodeForKey(propNode, "enum"); enum != nil && enum.Kind == yaml.SequenceNode && len(enum.Content) > 0 { - return fmt.Sprintf("\"%s\"", enum.Content[0].Value) + return fmt.Sprintf("%q", enum.Content[0].Value) } - + // Generate based on type and property name switch propType { case "string": @@ -742,4 +735,4 @@ func generateExampleValue(propName, propType string, propNode *yaml.Node) string default: return "\"example_value\"" } -} \ No newline at end of file +} From f2ed2c499909a90b52d634bb75ad4deba222cb8d Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 4 Aug 2025 13:58:37 -0600 Subject: [PATCH 07/32] add sec warning --- cli/cmd/docs/generate_project.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index b125b8777f9..36987d6085a 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -496,6 +496,10 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req doc.WriteString("- [**Slack**](#slack) - Slack data\n") doc.WriteString("- [**Local File**](#local_file) - Local file system\n\n") + doc.WriteString(":::warning Security Recommendation\n") + doc.WriteString("For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/build/credentials/) for complete setup instructions.\n") + doc.WriteString(":::\n\n") + doc.WriteString("## Connector Details\n\n") break } From d57d430b4ad239b3913a2de57c1cf48f88d1d09e Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 4 Aug 2025 14:59:35 -0600 Subject: [PATCH 08/32] motherduck live connector --- cli/cmd/docs/generate_project.go | 11 +++++++++-- runtime/parser/schema/connector.schema.yaml | 16 ++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index 36987d6085a..703922543ca 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -616,7 +616,12 @@ func generateConnectorExample(connectorType string, connectorDef *yaml.Node) str // Fallback: if driver wasn't found, use the connector type name if !driverAdded { - example.WriteString(fmt.Sprintf("driver: %s # Must be `%s` _(required)_\n\n", strings.ToLower(connectorType), strings.ToLower(connectorType))) + // Special case for MotherDuck which uses duckdb driver + if connectorType == "motherduck" { + example.WriteString("driver: duckdb # Must be `duckdb` _(required)_\n\n") + } else { + example.WriteString(fmt.Sprintf("driver: %s # Must be `%s` _(required)_\n\n", strings.ToLower(connectorType), strings.ToLower(connectorType))) + } } // Get all properties from the schema @@ -704,7 +709,7 @@ func generateExampleValue(propName, propType string, propNode *yaml.Node) string case strings.Contains(propName, "dsn"): return "\"postgresql://user:pass@localhost:5432/db\"" case strings.Contains(propName, "path"): - return "\"/path/to/file\"" + return "\"md:my_db\"" case strings.Contains(propName, "uri"): return "\"s3://bucket/path\"" case strings.Contains(propName, "endpoint"): @@ -723,6 +728,8 @@ func generateExampleValue(propName, propType string, propNode *yaml.Node) string return "\"MyExternalID\"" case strings.Contains(propName, "location"): return "\"s3://my-bucket/athena-output/\"" + case strings.Contains(propName, "init_sql"): + return "|\n INSTALL 'motherduck';\n LOAD 'motherduck';\n SET motherduck_token= '{{ .env.motherduck_token }}'" default: return "\"example_value\"" } diff --git a/runtime/parser/schema/connector.schema.yaml b/runtime/parser/schema/connector.schema.yaml index 1475074f897..b30fd1eb80a 100644 --- a/runtime/parser/schema/connector.schema.yaml +++ b/runtime/parser/schema/connector.schema.yaml @@ -349,18 +349,18 @@ definitions: properties: driver: type: string - description: Refers to the driver type and must be driver `motherduck` - const: motherduck - dsn: + description: Refers to the driver type and must be driver `duckdb` + const: duckdb + path: type: string - description: Data Source Name (DSN) specifying the MotherDuck connection endpoint - token: + description: Path to your MD database + init_sql: type: string - description: Authentication token for accessing MotherDuck (secret) + description: SQL executed during database initialization. required: - driver - - dsn - - token + - path + - init_sql mysql: type: object title: mysql From e5434eea995e4f843758241fe10c92a99cd97337 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 4 Aug 2025 15:02:05 -0600 Subject: [PATCH 09/32] remove local file from connector --- runtime/parser/schema/connector.schema.yaml | 38 ++++++++++----------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/runtime/parser/schema/connector.schema.yaml b/runtime/parser/schema/connector.schema.yaml index b30fd1eb80a..5b81e07685a 100644 --- a/runtime/parser/schema/connector.schema.yaml +++ b/runtime/parser/schema/connector.schema.yaml @@ -28,7 +28,7 @@ allOf: - $ref: '#/definitions/duckdb' - $ref: '#/definitions/gcs' - $ref: '#/definitions/https' - - $ref: '#/definitions/local_file' + # - $ref: '#/definitions/local_file' - $ref: '#/definitions/motherduck' - $ref: '#/definitions/mysql' - $ref: '#/definitions/pinot' @@ -324,24 +324,24 @@ definitions: required: - driver - path - local_file: - type: object - title: local_file - description: Configuration properties specific to the local_file - properties: - driver: - type: string - description: Refers to the driver type and must be driver `local_file` - const: local_file - dsn: - type: string - description: Data Source Name (DSN) indicating the file path or location of the local file - allow_host_access: - type: boolean - description: Flag to indicate if access to host-level file paths is permitted - required: - - driver - - dsn + # local_file: + # type: object + # title: local_file + # description: Configuration properties specific to the local_file + # properties: + # driver: + # type: string + # description: Refers to the driver type and must be driver `local_file` + # const: local_file + # sql: + # type: string + # description: SQL query to execute on the local file, file should exist in the local root data directory + # allow_host_access: + # type: boolean + # description: Flag to indicate if access to host-level file paths is permitted + # required: + # - driver + # - sql motherduck: type: object title: motherduck From cd0c8192624566476d2c3d323b03a2574abc0a49 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 4 Aug 2025 15:13:37 -0600 Subject: [PATCH 10/32] nit --- cli/cmd/docs/generate_project.go | 2 +- runtime/parser/schema/connector.schema.yaml | 36 ++++++++++----------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index 703922543ca..1365a9b6a15 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -617,7 +617,7 @@ func generateConnectorExample(connectorType string, connectorDef *yaml.Node) str // Fallback: if driver wasn't found, use the connector type name if !driverAdded { // Special case for MotherDuck which uses duckdb driver - if connectorType == "motherduck" { + if connectorType == "MotherDuck" { example.WriteString("driver: duckdb # Must be `duckdb` _(required)_\n\n") } else { example.WriteString(fmt.Sprintf("driver: %s # Must be `%s` _(required)_\n\n", strings.ToLower(connectorType), strings.ToLower(connectorType))) diff --git a/runtime/parser/schema/connector.schema.yaml b/runtime/parser/schema/connector.schema.yaml index 5b81e07685a..12fe1826fb8 100644 --- a/runtime/parser/schema/connector.schema.yaml +++ b/runtime/parser/schema/connector.schema.yaml @@ -43,7 +43,7 @@ allOf: definitions: athena: type: object - title: athena + title: Athena description: Configuration properties specific to the athena properties: driver: @@ -84,7 +84,7 @@ definitions: - driver azure: type: object - title: azure + title: Azure description: Configuration properties specific to the azure properties: driver: @@ -114,7 +114,7 @@ definitions: - azure_storage_bucket bigquery: type: object - title: bigquery + title: BigQuery description: Configuration properties specific to the bigquery properties: driver: @@ -134,7 +134,7 @@ definitions: - driver clickhouse: type: object - title: clickhouse + title: ClickHouse description: Configuration properties specific to the clickhouse properties: driver: @@ -202,7 +202,7 @@ definitions: - driver druid: type: object - title: druid + title: Druid description: Configuration properties specific to the druid properties: driver: @@ -241,7 +241,7 @@ definitions: - dsn duckdb: type: object - title: duckdb + title: DuckDB description: Configuration properties specific to the duckdb properties: driver: @@ -279,7 +279,7 @@ definitions: - driver gcs: type: object - title: gcs + title: GCS description: Configuration properties specific to the gcs properties: driver: @@ -306,7 +306,7 @@ definitions: - bucket https: type: object - title: https + title: HTTPS description: Configuration properties specific to the https properties: driver: @@ -344,7 +344,7 @@ definitions: # - sql motherduck: type: object - title: motherduck + title: MotherDuck description: Configuration properties specific to the motherduck properties: driver: @@ -363,7 +363,7 @@ definitions: - init_sql mysql: type: object - title: mysql + title: MySQL description: Configuration properties specific to the mysql properties: driver: @@ -395,7 +395,7 @@ definitions: - driver pinot: type: object - title: pinot + title: Pinot description: Configuration properties specific to the pinot properties: driver: @@ -439,7 +439,7 @@ definitions: - controller_host postgres: type: object - title: postgres + title: Postgres description: Configuration properties specific to the postgres properties: driver: @@ -471,7 +471,7 @@ definitions: - driver redshift: type: object - title: redshift + title: Redshift description: Configuration properties specific to the redshift properties: driver: @@ -506,7 +506,7 @@ definitions: - database s3: type: object - title: s3 + title: S3 description: Configuration properties specific to the s3 properties: driver: @@ -542,7 +542,7 @@ definitions: - bucket salesforce: type: object - title: salesforce + title: Salesforce description: Configuration properties specific to the salesforce properties: driver: @@ -570,7 +570,7 @@ definitions: - endpoint slack: type: object - title: slack + title: Slack description: Configuration properties specific to the slack properties: driver: @@ -585,7 +585,7 @@ definitions: - bot_token snowflake: type: object - title: snowflake + title: Snowflake description: Configuration properties specific to the snowflake properties: driver: @@ -603,7 +603,7 @@ definitions: - dsn sqlite: type: object - title: sqlite + title: SQLite description: Configuration properties specific to the sqlite properties: driver: From d3396ebe519ecdc22c3b9ba063c035e6a6390349 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 4 Aug 2025 15:36:48 -0600 Subject: [PATCH 11/32] matching file names --- docs/docs/hidden/yaml/advanced-models.md | 802 ++++++++++++++++++ docs/docs/hidden/yaml/alerts.md | 217 +++++ docs/docs/hidden/yaml/apis.md | 210 +++++ docs/docs/hidden/yaml/canvas-dashboards.md | 209 +++++ docs/docs/hidden/yaml/connectors.md | 742 ++++++++++++++++ docs/docs/hidden/yaml/explore-dashboards.md | 295 +++++++ docs/docs/hidden/yaml/index.md | 19 +- docs/docs/hidden/yaml/metrics-views.md | 186 ++++ docs/docs/hidden/yaml/project.md | 6 +- docs/docs/hidden/yaml/themes.md | 52 ++ ...chema.yaml => advanced-models.schema.yaml} | 38 +- .../{alert.schema.yaml => alerts.schema.yaml} | 4 +- .../{api.schema.yaml => apis.schema.yaml} | 4 +- ...ema.yaml => canvas-dashboards.schema.yaml} | 6 +- ...tor.schema.yaml => connectors.schema.yaml} | 4 +- ...ma.yaml => explore-dashboards.schema.yaml} | 4 +- ....schema.yaml => metrics_views.schema.yaml} | 4 +- runtime/parser/schema/project.schema.yaml | 18 +- runtime/parser/schema/rillyaml.schema.yaml | 4 +- .../{theme.schema.yaml => themes.schema.yaml} | 4 +- 20 files changed, 2770 insertions(+), 58 deletions(-) create mode 100644 docs/docs/hidden/yaml/advanced-models.md create mode 100644 docs/docs/hidden/yaml/alerts.md create mode 100644 docs/docs/hidden/yaml/apis.md create mode 100644 docs/docs/hidden/yaml/canvas-dashboards.md create mode 100644 docs/docs/hidden/yaml/connectors.md create mode 100644 docs/docs/hidden/yaml/explore-dashboards.md create mode 100644 docs/docs/hidden/yaml/metrics-views.md create mode 100644 docs/docs/hidden/yaml/themes.md rename runtime/parser/schema/{advanced-model.schema.yaml => advanced-models.schema.yaml} (96%) rename runtime/parser/schema/{alert.schema.yaml => alerts.schema.yaml} (99%) rename runtime/parser/schema/{api.schema.yaml => apis.schema.yaml} (99%) rename runtime/parser/schema/{canvas.schema.yaml => canvas-dashboards.schema.yaml} (97%) rename runtime/parser/schema/{connector.schema.yaml => connectors.schema.yaml} (99%) rename runtime/parser/schema/{explore.schema.yaml => explore-dashboards.schema.yaml} (99%) rename runtime/parser/schema/{metrics_view.schema.yaml => metrics_views.schema.yaml} (99%) rename runtime/parser/schema/{theme.schema.yaml => themes.schema.yaml} (98%) diff --git a/docs/docs/hidden/yaml/advanced-models.md b/docs/docs/hidden/yaml/advanced-models.md new file mode 100644 index 00000000000..e8487e7740d --- /dev/null +++ b/docs/docs/hidden/yaml/advanced-models.md @@ -0,0 +1,802 @@ +--- +note: GENERATED. DO NOT EDIT. +title: Advanced Models YAML +sidebar_position: 37 +--- + +## Properties + +### `type` + +_[string]_ - Refers to the resource type and must be `model` _(required)_ + +### `refresh` + +_[object]_ - Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying model data + + - **`cron`** - _[string]_ - A cron expression that defines the execution schedule + + - **`time_zone`** - _[string]_ - Time zone to interpret the schedule in (e.g., 'UTC', 'America/Los_Angeles'). + + - **`disable`** - _[boolean]_ - If true, disables the resource without deleting it. + + - **`ref_update`** - _[boolean]_ - If true, allows the resource to run when a dependency updates. + + - **`run_in_dev`** - _[boolean]_ - If true, allows the schedule to run in development mode. + +### `connector` + +_[object]_ - Connector YAML files define how Rill connects to external data sources and OLAP engines. Each connector specifies a driver type and its required connection parameters. + + + - **`type`** - _[string]_ - Refers to the resource type and must be `connector` _(required)_ + + - **`driver`** - _[string]_ - The type of connector, see [available connectors](#available-connector-types) (required) _(required)_ + +#### Option 1: Athena + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the athena + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `athena` _(required)_ + + - **`aws_access_key_id`** - _[string]_ - AWS Access Key ID used for authentication. Required when using static credentials directly or as base credentials for assuming a role. + + - **`aws_secret_access_key`** - _[string]_ - AWS Secret Access Key paired with the Access Key ID. Required when using static credentials directly or as base credentials for assuming a role. + + - **`aws_access_token`** - _[string]_ - AWS session token used with temporary credentials. Required only if the Access Key and Secret Key are part of a temporary session credentials. + + - **`role_arn`** - _[string]_ - ARN of the IAM role to assume. When specified, the SDK uses the base credentials to call STS AssumeRole and obtain temporary credentials scoped to this role. + + - **`role_session_name`** - _[string]_ - Session name to associate with the STS AssumeRole session. Used only if 'role_arn' is specified. Useful for identifying and auditing the session. + + - **`external_id`** - _[string]_ - External ID required by some roles when assuming them, typically for cross-account access. Used only if 'role_arn' is specified and the role's trust policy requires it. + + - **`workgroup`** - _[string]_ - Athena workgroup to use for query execution. Defaults to 'primary' if not specified. + + - **`output_location`** - _[string]_ - S3 URI where Athena query results should be stored (e.g., s3://your-bucket/athena/results/). Optional if the selected workgroup has a default result configuration. + + - **`aws_region`** - _[string]_ - AWS region where Athena and the result S3 bucket are located (e.g., us-east-1). Defaults to 'us-east-1' if not specified. + + - **`allow_host_access`** - _[boolean]_ - Allow the Athena client to access host environment configurations such as environment variables or local AWS credential files. Defaults to true, enabling use of credentials and settings from the host environment unless explicitly disabled. + +#### Option 2: Azure + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the azure + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `azure` _(required)_ + + - **`azure_storage_account`** - _[string]_ - Azure storage account name + + - **`azure_storage_key`** - _[string]_ - Azure storage access key + + - **`azure_storage_sas_token`** - _[string]_ - Optional azure SAS token for authentication + + - **`azure_storage_connection_string`** - _[string]_ - Optional azure connection string for storage account + + - **`azure_storage_bucket`** - _[string]_ - Name of the Azure Blob Storage container (equivalent to an S3 bucket) _(required)_ + + - **`allow_host_access`** - _[boolean]_ - Allow access to host environment configuration + +#### Option 3: BigQuery + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the bigquery + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `bigquery` _(required)_ + + - **`google_application_credentials`** - _[string]_ - Raw contents of the Google Cloud service account key (in JSON format) used for authentication. + + - **`project_id`** - _[string]_ - ID of the Google Cloud project to use for BigQuery operations. This can be omitted only if the project ID is included in the service account key. + + - **`allow_host_access`** - _[boolean]_ - Enable the BigQuery client to use credentials from the host environment when no service account JSON is provided. This includes Application Default Credentials from environment variables, local credential files, or the Google Compute Engine metadata server. Defaults to true, allowing seamless authentication in GCP environments. + +#### Option 4: ClickHouse + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the clickhouse + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `clickhouse` _(required)_ + + - **`managed`** - _[boolean]_ - `true` means Rill will provision the connector using the default provisioner. `false` disables automatic provisioning. + + - **`mode`** - _[string]_ - `read` - Controls the operation mode for the ClickHouse connection. Defaults to 'read' for safe operation with external databases. Set to 'readwrite' to enable model creation and table mutations. Note: When 'managed: true', this is automatically set to 'readwrite'. + + - **`dsn`** - _[string]_ - DSN(Data Source Name) for the ClickHouse connection + + - **`username`** - _[string]_ - Username for authentication + + - **`password`** - _[string]_ - Password for authentication + + - **`host`** - _[string]_ - Host where the ClickHouse instance is running + + - **`port`** - _[integer]_ - Port where the ClickHouse instance is accessible + + - **`database`** - _[string]_ - Name of the ClickHouse database within the cluster + + - **`ssl`** - _[boolean]_ - Indicates whether a secured SSL connection is required + + - **`cluster`** - _[string]_ - Cluster name, required for running distributed queries + + - **`log_queries`** - _[boolean]_ - Controls whether to log raw SQL queries + + - **`settings_override`** - _[string]_ - override the default settings used in queries. example `readonly = 1, session_timezone = 'UTC'` + + - **`embed_port`** - _[integer]_ - Port to run ClickHouse locally (0 for random port) + + - **`can_scale_to_zero`** - _[boolean]_ - Indicates if the database can scale to zero + + - **`max_open_conns`** - _[integer]_ - Maximum number of open connections to the database + + - **`max_idle_conns`** - _[integer]_ - Maximum number of idle connections in the pool + + - **`dial_timeout`** - _[string]_ - Timeout for dialing the ClickHouse server + + - **`conn_max_lifetime`** - _[string]_ - Maximum time a connection may be reused + + - **`read_timeout`** - _[string]_ - Maximum time for a connection to read data + +#### Option 5: Druid + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the druid + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `druid` _(required)_ + + - **`dsn`** - _[string]_ - Data Source Name (DSN) for connecting to Druid _(required)_ + + - **`username`** - _[string]_ - Username for authenticating with Druid + + - **`password`** - _[string]_ - Password for authenticating with Druid + + - **`host`** - _[string]_ - Hostname of the Druid coordinator or broker + + - **`port`** - _[integer]_ - Port number of the Druid service + + - **`ssl`** - _[boolean]_ - Enable SSL for secure connection + + - **`log_queries`** - _[boolean]_ - Log raw SQL queries sent to Druid + + - **`max_open_conns`** - _[integer]_ - Maximum number of open database connections (0 = default, -1 = unlimited) + + - **`skip_version_check`** - _[boolean]_ - Skip checking Druid version compatibility + +#### Option 6: DuckDB + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the duckdb + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `duckdb` _(required)_ + + - **`pool_size`** - _[integer]_ - Number of concurrent connections and queries allowed + + - **`allow_host_access`** - _[boolean]_ - Whether access to the local environment and file system is allowed + + - **`cpu`** - _[integer]_ - Number of CPU cores available to the database + + - **`memory_limit_gb`** - _[integer]_ - Amount of memory in GB available to the database + + - **`read_write_ratio`** - _[number]_ - Ratio of resources allocated to the read database; used to divide CPU and memory + + - **`init_sql`** - _[string]_ - is executed during database initialization. + + - **`conn_init_sql`** - _[string]_ - is executed when a new connection is initialized. + + - **`secrets`** - _[string]_ - Comma-separated list of other connector names to create temporary secrets for in DuckDB before executing a model. + + - **`log_queries`** - _[boolean]_ - Whether to log raw SQL queries executed through OLAP + +#### Option 7: GCS + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the gcs + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `gcs` _(required)_ + + - **`google_application_credentials`** - _[string]_ - Google Cloud credentials JSON string + + - **`bucket`** - _[string]_ - Name of gcs bucket _(required)_ + + - **`allow_host_access`** - _[boolean]_ - Allow access to host environment configuration + + - **`key_id`** - _[string]_ - Optional S3-compatible Key ID when used in compatibility mode + + - **`secret`** - _[string]_ - Optional S3-compatible Secret when used in compatibility mode + +#### Option 8: HTTPS + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the https + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `https` _(required)_ + + - **`path`** - _[string]_ - The full HTTPS URI to fetch data from _(required)_ + + - **`headers`** - _[object]_ - HTTP headers to include in the request + +#### Option 9: MotherDuck + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the motherduck + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `duckdb` _(required)_ + + - **`path`** - _[string]_ - Path to your MD database _(required)_ + + - **`init_sql`** - _[string]_ - SQL executed during database initialization. _(required)_ + +#### Option 10: MySQL + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the mysql + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `mysql` _(required)_ + + - **`dsn`** - _[string]_ - DSN(Data Source Name) for the mysql connection + + - **`host`** - _[string]_ - Hostname of the MySQL server + + - **`port`** - _[integer]_ - Port number for the MySQL server + + - **`database`** - _[string]_ - Name of the MySQL database + + - **`user`** - _[string]_ - Username for authentication + + - **`password`** - _[string]_ - Password for authentication + + - **`ssl_mode`** - _[string]_ - SSL mode can be DISABLED, PREFERRED or REQUIRED + +#### Option 11: Pinot + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the pinot + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `pinot` _(required)_ + + - **`dsn`** - _[string]_ - DSN(Data Source Name) for the Pinot connection _(required)_ + + - **`username`** - _[string]_ - Username for authenticating with Pinot + + - **`password`** - _[string]_ - Password for authenticating with Pinot + + - **`broker_host`** - _[string]_ - Hostname of the Pinot broker _(required)_ + + - **`broker_port`** - _[integer]_ - Port number for the Pinot broker + + - **`controller_host`** - _[string]_ - Hostname of the Pinot controller _(required)_ + + - **`controller_port`** - _[integer]_ - Port number for the Pinot controller + + - **`ssl`** - _[boolean]_ - Enable SSL connection to Pinot + + - **`log_queries`** - _[boolean]_ - Log raw SQL queries executed through Pinot + + - **`max_open_conns`** - _[integer]_ - Maximum number of open connections to the Pinot database + +#### Option 12: Postgres + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the postgres + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `postgres` _(required)_ + + - **`dsn`** - _[string]_ - DSN(Data Source Name) for the postgres connection + + - **`host`** - _[string]_ - Hostname of the Postgres server + + - **`port`** - _[string]_ - Port number for the Postgres server + + - **`dbname`** - _[string]_ - Name of the Postgres database + + - **`user`** - _[string]_ - Username for authentication + + - **`password`** - _[string]_ - Password for authentication + + - **`sslmode`** - _[string]_ - SSL mode can be disable, allow, prefer or require + +#### Option 13: Redshift + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the redshift + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `redshift` _(required)_ + + - **`aws_access_key_id`** - _[string]_ - AWS Access Key ID used for authenticating with Redshift. _(required)_ + + - **`aws_secret_access_key`** - _[string]_ - AWS Secret Access Key used for authenticating with Redshift. _(required)_ + + - **`aws_access_token`** - _[string]_ - AWS Session Token for temporary credentials (optional). + + - **`region`** - _[string]_ - AWS region where the Redshift cluster or workgroup is hosted (e.g., 'us-east-1'). + + - **`database`** - _[string]_ - Name of the Redshift database to query. _(required)_ + + - **`workgroup`** - _[string]_ - Workgroup name for Redshift Serverless, in case of provisioned Redshift clusters use 'cluster_identifier'. + + - **`cluster_identifier`** - _[string]_ - Cluster identifier for provisioned Redshift clusters, in case of Redshift Serverless use 'workgroup' . + +#### Option 14: S3 + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the s3 + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `s3` _(required)_ + + - **`aws_access_key_id`** - _[string]_ - AWS Access Key ID used for authentication + + - **`aws_secret_access_key`** - _[string]_ - AWS Secret Access Key used for authentication + + - **`aws_access_token`** - _[string]_ - Optional AWS session token for temporary credentials + + - **`bucket`** - _[string]_ - Name of s3 bucket _(required)_ + + - **`endpoint`** - _[string]_ - Optional custom endpoint URL for S3-compatible storage + + - **`region`** - _[string]_ - AWS region of the S3 bucket + + - **`allow_host_access`** - _[boolean]_ - Allow access to host environment configuration + + - **`retain_files`** - _[boolean]_ - Whether to retain intermediate files after processing + +#### Option 15: Salesforce + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the salesforce + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `salesforce` _(required)_ + + - **`username`** - _[string]_ - Salesforce account username _(required)_ + + - **`password`** - _[string]_ - Salesforce account password (secret) + + - **`key`** - _[string]_ - Authentication key for Salesforce (secret) + + - **`endpoint`** - _[string]_ - Salesforce API endpoint URL _(required)_ + + - **`client_id`** - _[string]_ - Client ID used for Salesforce OAuth authentication + +#### Option 16: Slack + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the slack + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `slack` _(required)_ + + - **`bot_token`** - _[string]_ - Bot token used for authenticating Slack API requests _(required)_ + +#### Option 17: Snowflake + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the snowflake + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `snowflake` _(required)_ + + - **`dsn`** - _[string]_ - DSN (Data Source Name) for the Snowflake connection _(required)_ + + - **`parallel_fetch_limit`** - _[integer]_ - Maximum number of concurrent fetches during query execution + +#### Option 18: SQLite + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the sqlite + + - **`driver`** - _[string]_ - Refers to the driver type and must be driver `sqlite` _(required)_ + + - **`dsn`** - _[string]_ - DSN(Data Source Name) for the sqlite connection _(required)_ + +### `sql` + +_[string]_ - Raw SQL query to run against source _(required)_ + +### `timeout` + +_[string]_ - The maximum time to wait for model ingestion + +### `incremental` + +_[boolean]_ - whether incremental modeling is required (optional) + +### `change_mode` + +_[string]_ - Configure how changes to the model specifications are applied (optional). 'reset' will drop and recreate the model automatically, 'manual' will require a manual full or incremental refresh to apply changes, and 'patch' will switch to the new logic without re-processing historical data (only applies for incremental models). + +### `state` + +_[oneOf]_ - Refers to the explicitly defined state of your model, cannot be used with partitions (optional) + +#### Option 1: SQL Query + +**Type:** _[object]_ + +**Description:** Executes a raw SQL query against the project's data models. + + - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ + + - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. + +#### Option 2: Metrics View Query + +**Type:** _[object]_ + +**Description:** Executes a SQL query that targets a defined metrics view. + + - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ + +#### Option 3: Custom API Call + +**Type:** _[object]_ + +**Description:** Calls a custom API defined in the project to compute data. + + - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ + + - **`args`** - _[object]_ - Arguments to pass to the custom API. + +#### Option 4: File Glob Query + +**Type:** _[object]_ + +**Description:** Uses a file-matching pattern (glob) to query data from a connector. + + - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ + + - **option 1** - _[string]_ - A simple file path/glob pattern as a string. + + - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. + + - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. + +#### Option 5: Resource Status Check + +**Type:** _[object]_ + +**Description:** Uses the status of a resource as data. + + - **`resource_status`** - _[object]_ - Based on resource status _(required)_ + + - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. + +### `partitions` + +_[oneOf]_ - Refers to the how your data is partitioned, cannot be used with state. (optional) + +#### Option 1: SQL Query + +**Type:** _[object]_ + +**Description:** Executes a raw SQL query against the project's data models. + + - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ + + - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. + +#### Option 2: Metrics View Query + +**Type:** _[object]_ + +**Description:** Executes a SQL query that targets a defined metrics view. + + - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ + +#### Option 3: Custom API Call + +**Type:** _[object]_ + +**Description:** Calls a custom API defined in the project to compute data. + + - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ + + - **`args`** - _[object]_ - Arguments to pass to the custom API. + +#### Option 4: File Glob Query + +**Type:** _[object]_ + +**Description:** Uses a file-matching pattern (glob) to query data from a connector. + + - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ + + - **option 1** - _[string]_ - A simple file path/glob pattern as a string. + + - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. + + - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. + +#### Option 5: Resource Status Check + +**Type:** _[object]_ + +**Description:** Uses the status of a resource as data. + + - **`resource_status`** - _[object]_ - Based on resource status _(required)_ + + - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. + +### `materialize` + +_[boolean]_ - models will be materialized in olap + +### `partitions_watermark` + +_[string]_ - Refers to a customizable timestamp that can be set to check if an object has been updated (optional). + +### `partitions_concurrency` + +_[integer]_ - Refers to the number of concurrent partitions that can be read at the same time (optional). + +### `stage` + +_[object]_ - in the case of staging models, where an input source does not support direct write to the output and a staging table is required + + - **`connector`** - _[string]_ - Refers to the connector type for the staging table _(required)_ + +### `output` + +_[object]_ - to define the properties of output + + - **`table`** - _[string]_ - Name of the output table. If not specified, the model name is used. + + - **`materialize`** - _[boolean]_ - Whether to materialize the model as a table or view + + - **`connector`** - _[string]_ - Refers to the connector type for the output table. Can be `clickhouse` or `duckdb` and their named connector + + - **`incremental_strategy`** - _[string]_ - Strategy to use for incremental updates. Can be 'append', 'merge' or 'partition_overwrite' + + - **`unique_key`** - _[array of string]_ - List of columns that uniquely identify a row for merge strategy + + - **`partition_by`** - _[string]_ - Column or expression to partition the table by + + **Additional properties for `output` when `connector` is `clickhouse`** + + - **`type`** - _[string]_ - Type to materialize the model into. Can be 'TABLE', 'VIEW' or 'DICTIONARY' + + - **`columns`** - _[string]_ - Column names and types. Can also include indexes. If unspecified, detected from the query. + + - **`engine_full`** - _[string]_ - Full engine definition in SQL format. Can include partition keys, order, TTL, etc. + + - **`engine`** - _[string]_ - Table engine to use. Default is MergeTree + + - **`order_by`** - _[string]_ - ORDER BY clause. + + - **`partition_by`** - _[string]_ - Partition BY clause. + + - **`primary_key`** - _[string]_ - PRIMARY KEY clause. + + - **`sample_by`** - _[string]_ - SAMPLE BY clause. + + - **`ttl`** - _[string]_ - TTL settings for the table or columns. + + - **`table_settings`** - _[string]_ - Table-specific settings. + + - **`query_settings`** - _[string]_ - Settings used in insert/create table as select queries. + + - **`distributed_settings`** - _[string]_ - Settings for distributed table. + + - **`distributed_sharding_key`** - _[string]_ - Sharding key for distributed table. + + - **`dictionary_source_user`** - _[string]_ - User for accessing the source dictionary table (used if type is DICTIONARY). + + - **`dictionary_source_password`** - _[string]_ - Password for the dictionary source user. + +## Common Properties + +### `name` + +_[string]_ - Name is usually inferred from the filename, but can be specified manually. + +### `refs` + +_[array of string]_ - List of resource references + +### `dev` + +_[object]_ - Overrides any properties in development environment. + +### `prod` + +_[object]_ - Overrides any properties in production environment. + +## Additional properties when `connector` is [`athena`](./connectors#athena) + +### `output_location` + +_[string]_ - Output location for query results in S3. + +### `workgroup` + +_[string]_ - AWS Athena workgroup to use for queries. + +### `region` + +_[string]_ - AWS region to connect to Athena and the output location. + +## Additional properties when `connector` is [`azure`](./connectors#azure) + +### `path` + +_[string]_ - Path to the source + +### `account` + +_[string]_ - Account identifier + +### `uri` + +_[string]_ - Source URI + +### `extract` + +_[object]_ - Arbitrary key-value pairs for extraction settings + +### `glob` + +_[object]_ - Settings related to glob file matching. + + - **`max_total_size`** - _[integer]_ - Maximum total size (in bytes) matched by glob + + - **`max_objects_matched`** - _[integer]_ - Maximum number of objects matched by glob + + - **`max_objects_listed`** - _[integer]_ - Maximum number of objects listed in glob + + - **`page_size`** - _[integer]_ - Page size for glob listing + +### `batch_size` + +_[string]_ - Size of a batch (e.g., '100MB') + +## Additional properties when `connector` is [`bigquery`](./connectors#bigquery) + +### `project_id` + +_[string]_ - ID of the BigQuery project. + +## Additional properties when `connector` is [`duckdb`](./connectors#duckdb) + +### `path` + +_[string]_ - Path to the data source. + +### `format` + +_[string]_ - Format of the data source (e.g., csv, json, parquet). + +### `pre_exec` + +_[string]_ - refers to SQL queries to run before the main query, available for DuckDB-based models. _(optional)_. Ensure `pre_exec` queries are idempotent. Use `IF NOT EXISTS` statements when applicable. + +### `post_exec` + +_[string]_ - refers to a SQL query that is run after the main query, available for DuckDB-based models. _(optional)_. Ensure `post_exec` queries are idempotent. Use `IF EXISTS` statements when applicable. + +```yaml +pre_exec: ATTACH IF NOT EXISTS 'dbname=postgres host=localhost port=5432 user=postgres password=postgres' AS postgres_db (TYPE POSTGRES); +sql: SELECT * FROM postgres_query('postgres_db', 'SELECT * FROM USERS') +post_exec: DETACH DATABASE IF EXISTS postgres_db +``` + +## Additional properties when `connector` is [`gcs`](./connectors#gcs) + +### `path` + +_[string]_ - Path to the source + +### `uri` + +_[string]_ - Source URI + +### `extract` + +_[object]_ - key-value pairs for extraction settings + +### `glob` + +_[object]_ - Settings related to glob file matching. + + - **`max_total_size`** - _[integer]_ - Maximum total size (in bytes) matched by glob + + - **`max_objects_matched`** - _[integer]_ - Maximum number of objects matched by glob + + - **`max_objects_listed`** - _[integer]_ - Maximum number of objects listed in glob + + - **`page_size`** - _[integer]_ - Page size for glob listing + +### `batch_size` + +_[string]_ - Size of a batch (e.g., '100MB') + +## Additional properties when `connector` is [`redshift`](./connectors#redshift) + +### `output_location` + +_[string]_ - S3 location where query results are stored. + +### `workgroup` + +_[string]_ - Redshift Serverless workgroup to use. + +### `database` + +_[string]_ - Name of the Redshift database. + +### `cluster_identifier` + +_[string]_ - Identifier of the Redshift cluster. + +### `role_arn` + +_[string]_ - ARN of the IAM role to assume for Redshift access. + +### `region` + +_[string]_ - AWS region of the Redshift deployment. + +## Additional properties when `connector` is [`s3`](./connectors#s3) + +### `region` + +_[string]_ - AWS region + +### `endpoint` + +_[string]_ - AWS Endpoint + +### `path` + +_[string]_ - Path to the source + +### `uri` + +_[string]_ - Source URI + +### `extract` + +_[object]_ - key-value pairs for extraction settings + +### `glob` + +_[object]_ - Settings related to glob file matching. + + - **`max_total_size`** - _[integer]_ - Maximum total size (in bytes) matched by glob + + - **`max_objects_matched`** - _[integer]_ - Maximum number of objects matched by glob + + - **`max_objects_listed`** - _[integer]_ - Maximum number of objects listed in glob + + - **`page_size`** - _[integer]_ - Page size for glob listing + +### `batch_size` + +_[string]_ - Size of a batch (e.g., '100MB') + +## Additional properties when `connector` is [`salesforce`](./connector#salesforce) + +### `soql` + +_[string]_ - SOQL query to execute against the Salesforce instance. + +### `sobject` + +_[string]_ - Salesforce object (e.g., Account, Contact) targeted by the query. + +### `queryAll` + +_[boolean]_ - Whether to include deleted and archived records in the query (uses queryAll API). \ No newline at end of file diff --git a/docs/docs/hidden/yaml/alerts.md b/docs/docs/hidden/yaml/alerts.md new file mode 100644 index 00000000000..f3719eded91 --- /dev/null +++ b/docs/docs/hidden/yaml/alerts.md @@ -0,0 +1,217 @@ +--- +note: GENERATED. DO NOT EDIT. +title: Alerts YAML +sidebar_position: 31 +--- + +Along with alertings at the dashboard level and can be created via the UI, there might be more extensive alerting that you might want to develop and can be done so the an alert.yaml. When creating an alert via a YAML file, you'll see this denoted in the UI as `Created through code`. + +## Properties + +### `type` + +_[string]_ - Refers to the resource type and must be `alert` _(required)_ + +### `display_name` + +_[string]_ - Refers to the display name for the alert + +### `refresh` + +_[object]_ - Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying data _(required)_ + + - **`cron`** - _[string]_ - A cron expression that defines the execution schedule + + - **`time_zone`** - _[string]_ - Time zone to interpret the schedule in (e.g., 'UTC', 'America/Los_Angeles'). + + - **`disable`** - _[boolean]_ - If true, disables the resource without deleting it. + + - **`ref_update`** - _[boolean]_ - If true, allows the resource to run when a dependency updates. + + - **`run_in_dev`** - _[boolean]_ - If true, allows the schedule to run in development mode. + +### `intervals` + +_[object]_ - define the interval of the alert to check + + - **`duration`** - _[string]_ - a valid ISO8601 duration to define the interval duration + + - **`limit`** - _[integer]_ - maximum number of intervals to check for on invocation + + - **`check_unclosed`** - _[boolean]_ - boolean, whether unclosed intervals should be checked + +### `watermark` + +_[string]_ - Specifies how the watermark is determined for incremental processing. Use 'trigger_time' to set it at runtime or 'inherit' to use the upstream model's watermark. + +### `timeout` + +_[string]_ - define the timeout of the alert in seconds (optional). + +### `data` + +_[oneOf]_ - Specifies one of the options to retrieve or compute the data used by alert _(required)_ + +#### Option 1: SQL Query + +**Type:** _[object]_ + +**Description:** Executes a raw SQL query against the project's data models. + + - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ + + - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. + +#### Option 2: Metrics View Query + +**Type:** _[object]_ + +**Description:** Executes a SQL query that targets a defined metrics view. + + - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ + +#### Option 3: Custom API Call + +**Type:** _[object]_ + +**Description:** Calls a custom API defined in the project to compute data. + + - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ + + - **`args`** - _[object]_ - Arguments to pass to the custom API. + +#### Option 4: File Glob Query + +**Type:** _[object]_ + +**Description:** Uses a file-matching pattern (glob) to query data from a connector. + + - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ + + - **option 1** - _[string]_ - A simple file path/glob pattern as a string. + + - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. + + - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. + +#### Option 5: Resource Status Check + +**Type:** _[object]_ + +**Description:** Uses the status of a resource as data. + + - **`resource_status`** - _[object]_ - Based on resource status _(required)_ + + - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. + +### `for` + +_[oneOf]_ - Specifies how user identity or attributes should be evaluated for security policy enforcement. + +#### Option 1 + +**Type:** _[object]_ + +**Description:** Specifies a unique user identifier for applying security policies. + + - **`user_id`** - _[string]_ - The unique user ID used to evaluate security policies. _(required)_ + +#### Option 2 + +**Type:** _[object]_ + +**Description:** Specifies a user's email address for applying security policies. + + - **`user_email`** - _[string]_ - The user's email address used to evaluate security policies. _(required)_ + +#### Option 3 + +**Type:** _[object]_ + +**Description:** Specifies a set of arbitrary user attributes for applying security policies. + + - **`attributes`** - _[object]_ - A dictionary of user attributes used to evaluate security policies. _(required)_ + +### `on_recover` + +_[boolean]_ - Send an alert when a previously failing alert recovers. Defaults to false. + +### `on_fail` + +_[boolean]_ - Send an alert when a failure occurs. Defaults to true. + +### `on_error` + +_[boolean]_ - Send an alert when an error occurs during evaluation. Defaults to false. + +### `renotify` + +_[boolean]_ - Enable repeated notifications for unresolved alerts. Defaults to false. + +### `renotify_after` + +_[string]_ - Defines the re-notification interval for the alert (e.g., '10m','24h'), equivalent to snooze duration in UI, defaults to 'Off' + +### `notify` + +_[object]_ - Defines how and where to send notifications. At least one method (email or Slack) is required. _(required)_ + + - **`email`** - _[object]_ - Send notifications via email. + + - **`recipients`** - _[array of string]_ - An array of email addresses to notify. _(required)_ + + - **`slack`** - _[object]_ - Send notifications via Slack. + + - **`users`** - _[array of string]_ - An array of Slack user IDs to notify. + + - **`channels`** - _[array of string]_ - An array of Slack channel IDs to notify. + + - **`webhooks`** - _[array of string]_ - An array of Slack webhook URLs to send notifications to. + +### `annotations` + +_[object]_ - Key value pair used for annotations + +## Common Properties + +### `name` + +_[string]_ - Name is usually inferred from the filename, but can be specified manually. + +### `refs` + +_[array of string]_ - List of resource references + +### `dev` + +_[object]_ - Overrides any properties in development environment. + +### `prod` + +_[object]_ - Overrides any properties in production environment. + +## Examples + +```yaml +# Example: To send alert when data lags by more than 1 day to slack channel #rill-cloud-alerts +type: alert +display_name: Data lags by more than 1 day +# Check the alert every hour. +refresh: + cron: 0 * * * * +# Query that returns non-empty results if the metrics lag by more than 1 day. +data: + sql: |- + SELECT * + FROM + ( + SELECT MAX(event_time) AS max_time + FROM rill_metrics_model + ) + WHERE max_time < NOW() - INTERVAL '1 day' +# Send notifications in Slack. +notify: + slack: + channels: + - '#rill-cloud-alerts' +``` \ No newline at end of file diff --git a/docs/docs/hidden/yaml/apis.md b/docs/docs/hidden/yaml/apis.md new file mode 100644 index 00000000000..729972b1117 --- /dev/null +++ b/docs/docs/hidden/yaml/apis.md @@ -0,0 +1,210 @@ +--- +note: GENERATED. DO NOT EDIT. +title: APIs YAML +sidebar_position: 32 +--- + +In your Rill project directory, create a new file name `.yaml` in the `apis` directory containing a custom API definition. See comprehensive documentation on how to define and use [custom APIs](/integrate/custom-apis/index.md) + +## Properties + +### `type` + +_[string]_ - Refers to the resource type and must be `api` _(required)_ + +### `openapi` + +_[object]_ - OpenAPI specification for the API endpoint + + - **`summary`** - _[string]_ - A brief description of what the API endpoint does + + - **`parameters`** - _[array of object]_ - List of parameters that the API endpoint accepts + + - **`request_schema`** - _[object]_ - JSON schema for the request body (use nested YAML instead of a JSON string) + + - **`response_schema`** - _[object]_ - JSON schema for the response body (use nested YAML instead of a JSON string) + +### `security` + +_[object]_ - Defines security rules and access control policies for resources + + - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + +#### Option 1 + +**Type:** _[string]_ + +**Description:** SQL expression that evaluates to a boolean to determine access + +#### Option 2 + +**Type:** _[boolean]_ + +**Description:** Direct boolean value to allow or deny access + + - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause + + - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded + + - **`if`** - _[string]_ - Expression to decide if the column should be included or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ + + - **`names`** - _[anyOf]_ - List of fields to include. Should match the name of one of the dashboard's dimensions or measures _(required)_ + + - **option 1** - _[array of string]_ - List of specific field names to include + + - **option 2** - _[string]_ - Wildcard '*' to include all fields + + - **`exclude`** - _[array of object]_ - List of dimension or measure names to exclude from the dashboard. If exclude is defined all other dimensions and measures are included + + - **`if`** - _[string]_ - Expression to decide if the column should be excluded or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ + + - **`names`** - _[anyOf]_ - List of fields to exclude. Should match the name of one of the dashboard's dimensions or measures _(required)_ + + - **option 1** - _[array of string]_ - List of specific field names to exclude + + - **option 2** - _[string]_ - Wildcard '*' to exclude all fields + + - **`rules`** - _[array of object]_ - List of detailed security rules that can be used to define complex access control policies + + - **`type`** - _[string]_ - Type of security rule - access (overall access), field_access (field-level access), or row_filter (row-level filtering) _(required)_ + + - **`action`** - _[string]_ - Whether to allow or deny access for this rule + + - **`if`** - _[string]_ - Conditional expression that determines when this rule applies. Must be a valid SQL expression that evaluates to a boolean + + - **`names`** - _[array of string]_ - List of field names this rule applies to (for field_access type rules) + + - **`all`** - _[boolean]_ - When true, applies the rule to all fields (for field_access type rules) + + - **`sql`** - _[string]_ - SQL expression for row filtering (for row_filter type rules) + +### `skip_nested_security` + +_[boolean]_ - Flag to control security inheritance + +## One of Properties Options +- [SQL Query](#sql-query) +- [Metrics View Query](#metrics-view-query) +- [Custom API Call](#custom-api-call) +- [File Glob Query](#file-glob-query) +- [Resource Status Check](#resource-status-check) + +#### Option 1: SQL Query + +**Type:** _[object]_ + +**Description:** Executes a raw SQL query against the project's data models. + +## SQL Query + +Executes a raw SQL query against the project's data models. + +### `sql` + +_[string]_ - Raw SQL query to run against existing models in the project. _(required)_ + +### `connector` + +_[string]_ - specifies the connector to use when running SQL or glob queries. + +#### Option 2: Metrics View Query + +**Type:** _[object]_ + +**Description:** Executes a SQL query that targets a defined metrics view. + +## Metrics View Query + +Executes a SQL query that targets a defined metrics view. + +### `metrics_sql` + +_[string]_ - SQL query that targets a metrics view in the project _(required)_ + +#### Option 3: Custom API Call + +**Type:** _[object]_ + +**Description:** Calls a custom API defined in the project to compute data. + +## Custom API Call + +Calls a custom API defined in the project to compute data. + +### `api` + +_[string]_ - Name of a custom API defined in the project. _(required)_ + +### `args` + +_[object]_ - Arguments to pass to the custom API. + +#### Option 4: File Glob Query + +**Type:** _[object]_ + +**Description:** Uses a file-matching pattern (glob) to query data from a connector. + +## File Glob Query + +Uses a file-matching pattern (glob) to query data from a connector. + +### `glob` + +_[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ + + - **option 1** - _[string]_ - A simple file path/glob pattern as a string. + + - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. + +### `connector` + +_[string]_ - Specifies the connector to use with the glob input. + +#### Option 5: Resource Status Check + +**Type:** _[object]_ + +**Description:** Uses the status of a resource as data. + +## Resource Status Check + +Uses the status of a resource as data. + +### `resource_status` + +_[object]_ - Based on resource status _(required)_ + + - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. + +## Common Properties + +### `name` + +_[string]_ - Name is usually inferred from the filename, but can be specified manually. + +### `refs` + +_[array of string]_ - List of resource references + +### `dev` + +_[object]_ - Overrides any properties in development environment. + +### `prod` + +_[object]_ - Overrides any properties in production environment. + +## Examples + +```yaml +# Example: This api returns the top 10 authors by net line changes since the specified date provided in the arguments. +type: api +name: metrics_view_api +metrics_sql: |- + SELECT author_name, net_line_changes + FROM advanced_metrics_view + where author_date > '{{ .args.date }}' + order by net_line_changes DESC + limit 10 +``` \ No newline at end of file diff --git a/docs/docs/hidden/yaml/canvas-dashboards.md b/docs/docs/hidden/yaml/canvas-dashboards.md new file mode 100644 index 00000000000..694b9a05730 --- /dev/null +++ b/docs/docs/hidden/yaml/canvas-dashboards.md @@ -0,0 +1,209 @@ +--- +note: GENERATED. DO NOT EDIT. +title: Canvas Dashboards YAML +sidebar_position: 33 +--- + +In your Rill project directory, create a canvas dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. + +## Properties + +### `type` + +_[string]_ - Refers to the resource type and must be `canvas` _(required)_ + +### `display_name` + +_[string]_ - Refers to the display name for the canvas + +### `banner` + +_[string]_ - Refers to the custom banner displayed at the header of an Canvas dashboard + +### `max_width` + +_[integer]_ - Max width in pixels of the canvas + +### `gap_x` + +_[integer]_ - Horizontal gap in pixels of the canvas + +### `gap_y` + +_[integer]_ - Vertical gap in pixels of the canvas + +### `theme` + +_[oneOf]_ - Name of the theme to use. Only one of theme and embedded_theme can be set. + +#### Option 1 + +**Type:** _[string]_ + +**Description:** Name of an existing theme to apply to the dashboard + +#### Option 2 + +**Type:** _[object]_ + +**Description:** Inline theme configuration. + + - **`colors`** - _[object]_ - Used to override the dashboard colors. Either primary or secondary color must be provided. + + - **`primary`** - _[string]_ - Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). + + - **`secondary`** - _[string]_ - Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. + +### `allow_custom_time_range` + +_[boolean]_ - Defaults to true, when set to false it will hide the ability to set a custom time range for the user. + +### `time_ranges` + +_[array of oneOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' + +#### Option 1 + +**Type:** _[string]_ + +**Description:** a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection + +#### Option 2 + +**Type:** _[object]_ + +**Description:** Object containing time range and comparison configuration + + - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ + + - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + +#### Option 1 + +**Type:** _[string]_ + +**Description:** Offset string only (range is inferred) + +#### Option 2 + +**Type:** _[object]_ + +**Description:** Object containing offset and range configuration for time comparison + + - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) + + - **`range`** - _[string]_ - Custom time range for comparison period + +### `time_zones` + +_[array of string]_ - Refers to the time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) + +### `filters` + +_[object]_ - Indicates if filters should be enabled for the canvas. + + - **`enable`** - _[boolean]_ - Toggles filtering functionality for the canvas dashboard. + +### `defaults` + +_[object]_ - Preset UI state to show by default + + - **`time_range`** - _[string]_ - Default time range to display when the dashboard loads + + - **`comparison_mode`** - _[string]_ - Default comparison mode for metrics (none, time, or dimension) + + - **`comparison_dimension`** - _[string]_ - Default dimension to use for comparison when comparison_mode is 'dimension' + +### `variables` + +_[array of object]_ - Variables that can be used in the canvas + + - **`name`** - _[string]_ - Unique identifier for the variable _(required)_ + + - **`type`** - _[string]_ - Data type of the variable (e.g., string, number, boolean) _(required)_ + + - **`value`** - _[string, number, boolean, object, array]_ - Default value for the variable. Can be any valid JSON value type + +### `rows` + +_[array of object]_ - Refers to all of the rows displayed on the Canvas _(required)_ + + - **`height`** - _[string]_ - Height of the row in px + + - **`items`** - _[array of object]_ - List of components to display in the row + + - **`component`** - _[string]_ - Name of the component to display + + - **`width`** - _[string, integer]_ - Width of the component (can be a number or string with unit) + +### `security` + +_[object]_ - Defines security rules and access control policies for resources + + - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + +#### Option 1 + +**Type:** _[string]_ + +**Description:** SQL expression that evaluates to a boolean to determine access + +#### Option 2 + +**Type:** _[boolean]_ + +**Description:** Direct boolean value to allow or deny access + + - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause + + - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded + + - **`if`** - _[string]_ - Expression to decide if the column should be included or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ + + - **`names`** - _[anyOf]_ - List of fields to include. Should match the name of one of the dashboard's dimensions or measures _(required)_ + + - **option 1** - _[array of string]_ - List of specific field names to include + + - **option 2** - _[string]_ - Wildcard '*' to include all fields + + - **`exclude`** - _[array of object]_ - List of dimension or measure names to exclude from the dashboard. If exclude is defined all other dimensions and measures are included + + - **`if`** - _[string]_ - Expression to decide if the column should be excluded or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ + + - **`names`** - _[anyOf]_ - List of fields to exclude. Should match the name of one of the dashboard's dimensions or measures _(required)_ + + - **option 1** - _[array of string]_ - List of specific field names to exclude + + - **option 2** - _[string]_ - Wildcard '*' to exclude all fields + + - **`rules`** - _[array of object]_ - List of detailed security rules that can be used to define complex access control policies + + - **`type`** - _[string]_ - Type of security rule - access (overall access), field_access (field-level access), or row_filter (row-level filtering) _(required)_ + + - **`action`** - _[string]_ - Whether to allow or deny access for this rule + + - **`if`** - _[string]_ - Conditional expression that determines when this rule applies. Must be a valid SQL expression that evaluates to a boolean + + - **`names`** - _[array of string]_ - List of field names this rule applies to (for field_access type rules) + + - **`all`** - _[boolean]_ - When true, applies the rule to all fields (for field_access type rules) + + - **`sql`** - _[string]_ - SQL expression for row filtering (for row_filter type rules) + +## Common Properties + +### `name` + +_[string]_ - Name is usually inferred from the filename, but can be specified manually. + +### `refs` + +_[array of string]_ - List of resource references + +### `dev` + +_[object]_ - Overrides any properties in development environment. + +### `prod` + +_[object]_ - Overrides any properties in production environment. \ No newline at end of file diff --git a/docs/docs/hidden/yaml/connectors.md b/docs/docs/hidden/yaml/connectors.md new file mode 100644 index 00000000000..55f65d385d0 --- /dev/null +++ b/docs/docs/hidden/yaml/connectors.md @@ -0,0 +1,742 @@ +--- +note: GENERATED. DO NOT EDIT. +title: Connectors YAML +sidebar_position: 34 +--- + +Connector YAML files define how Rill connects to external data sources and OLAP engines. Each connector specifies a driver type and its required connection parameters. + + +## Properties + +### `type` + +_[string]_ - Refers to the resource type and must be `connector` _(required)_ + +### `driver` + +_[string]_ - The type of connector, see [available connectors](#available-connector-types) (required) _(required)_ + +## One of Properties Options +- [Athena](#athena) +- [Azure](#azure) +- [BigQuery](#bigquery) +- [ClickHouse](#clickhouse) +- [Druid](#druid) +- [DuckDB](#duckdb) +- [GCS](#gcs) +- [HTTPS](#https) +- [MotherDuck](#motherduck) +- [MySQL](#mysql) +- [Pinot](#pinot) +- [Postgres](#postgres) +- [Redshift](#redshift) +- [S3](#s3) +- [Salesforce](#salesforce) +- [Slack](#slack) +- [Snowflake](#snowflake) +- [SQLite](#sqlite) + +#### Option 1: Athena + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the athena + +## Athena + +Configuration properties specific to the athena + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `athena` _(required)_ + +### `aws_access_key_id` + +_[string]_ - AWS Access Key ID used for authentication. Required when using static credentials directly or as base credentials for assuming a role. + +### `aws_secret_access_key` + +_[string]_ - AWS Secret Access Key paired with the Access Key ID. Required when using static credentials directly or as base credentials for assuming a role. + +### `aws_access_token` + +_[string]_ - AWS session token used with temporary credentials. Required only if the Access Key and Secret Key are part of a temporary session credentials. + +### `role_arn` + +_[string]_ - ARN of the IAM role to assume. When specified, the SDK uses the base credentials to call STS AssumeRole and obtain temporary credentials scoped to this role. + +### `role_session_name` + +_[string]_ - Session name to associate with the STS AssumeRole session. Used only if 'role_arn' is specified. Useful for identifying and auditing the session. + +### `external_id` + +_[string]_ - External ID required by some roles when assuming them, typically for cross-account access. Used only if 'role_arn' is specified and the role's trust policy requires it. + +### `workgroup` + +_[string]_ - Athena workgroup to use for query execution. Defaults to 'primary' if not specified. + +### `output_location` + +_[string]_ - S3 URI where Athena query results should be stored (e.g., s3://your-bucket/athena/results/). Optional if the selected workgroup has a default result configuration. + +### `aws_region` + +_[string]_ - AWS region where Athena and the result S3 bucket are located (e.g., us-east-1). Defaults to 'us-east-1' if not specified. + +### `allow_host_access` + +_[boolean]_ - Allow the Athena client to access host environment configurations such as environment variables or local AWS credential files. Defaults to true, enabling use of credentials and settings from the host environment unless explicitly disabled. + +#### Option 2: Azure + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the azure + +## Azure + +Configuration properties specific to the azure + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `azure` _(required)_ + +### `azure_storage_account` + +_[string]_ - Azure storage account name + +### `azure_storage_key` + +_[string]_ - Azure storage access key + +### `azure_storage_sas_token` + +_[string]_ - Optional azure SAS token for authentication + +### `azure_storage_connection_string` + +_[string]_ - Optional azure connection string for storage account + +### `azure_storage_bucket` + +_[string]_ - Name of the Azure Blob Storage container (equivalent to an S3 bucket) _(required)_ + +### `allow_host_access` + +_[boolean]_ - Allow access to host environment configuration + +#### Option 3: BigQuery + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the bigquery + +## BigQuery + +Configuration properties specific to the bigquery + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `bigquery` _(required)_ + +### `google_application_credentials` + +_[string]_ - Raw contents of the Google Cloud service account key (in JSON format) used for authentication. + +### `project_id` + +_[string]_ - ID of the Google Cloud project to use for BigQuery operations. This can be omitted only if the project ID is included in the service account key. + +### `allow_host_access` + +_[boolean]_ - Enable the BigQuery client to use credentials from the host environment when no service account JSON is provided. This includes Application Default Credentials from environment variables, local credential files, or the Google Compute Engine metadata server. Defaults to true, allowing seamless authentication in GCP environments. + +#### Option 4: ClickHouse + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the clickhouse + +## ClickHouse + +Configuration properties specific to the clickhouse + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `clickhouse` _(required)_ + +### `managed` + +_[boolean]_ - `true` means Rill will provision the connector using the default provisioner. `false` disables automatic provisioning. + +### `mode` + +_[string]_ - `read` - Controls the operation mode for the ClickHouse connection. Defaults to 'read' for safe operation with external databases. Set to 'readwrite' to enable model creation and table mutations. Note: When 'managed: true', this is automatically set to 'readwrite'. + +### `dsn` + +_[string]_ - DSN(Data Source Name) for the ClickHouse connection + +### `username` + +_[string]_ - Username for authentication + +### `password` + +_[string]_ - Password for authentication + +### `host` + +_[string]_ - Host where the ClickHouse instance is running + +### `port` + +_[integer]_ - Port where the ClickHouse instance is accessible + +### `database` + +_[string]_ - Name of the ClickHouse database within the cluster + +### `ssl` + +_[boolean]_ - Indicates whether a secured SSL connection is required + +### `cluster` + +_[string]_ - Cluster name, required for running distributed queries + +### `log_queries` + +_[boolean]_ - Controls whether to log raw SQL queries + +### `settings_override` + +_[string]_ - override the default settings used in queries. example `readonly = 1, session_timezone = 'UTC'` + +### `embed_port` + +_[integer]_ - Port to run ClickHouse locally (0 for random port) + +### `can_scale_to_zero` + +_[boolean]_ - Indicates if the database can scale to zero + +### `max_open_conns` + +_[integer]_ - Maximum number of open connections to the database + +### `max_idle_conns` + +_[integer]_ - Maximum number of idle connections in the pool + +### `dial_timeout` + +_[string]_ - Timeout for dialing the ClickHouse server + +### `conn_max_lifetime` + +_[string]_ - Maximum time a connection may be reused + +### `read_timeout` + +_[string]_ - Maximum time for a connection to read data + +#### Option 5: Druid + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the druid + +## Druid + +Configuration properties specific to the druid + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `druid` _(required)_ + +### `dsn` + +_[string]_ - Data Source Name (DSN) for connecting to Druid _(required)_ + +### `username` + +_[string]_ - Username for authenticating with Druid + +### `password` + +_[string]_ - Password for authenticating with Druid + +### `host` + +_[string]_ - Hostname of the Druid coordinator or broker + +### `port` + +_[integer]_ - Port number of the Druid service + +### `ssl` + +_[boolean]_ - Enable SSL for secure connection + +### `log_queries` + +_[boolean]_ - Log raw SQL queries sent to Druid + +### `max_open_conns` + +_[integer]_ - Maximum number of open database connections (0 = default, -1 = unlimited) + +### `skip_version_check` + +_[boolean]_ - Skip checking Druid version compatibility + +#### Option 6: DuckDB + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the duckdb + +## DuckDB + +Configuration properties specific to the duckdb + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `duckdb` _(required)_ + +### `pool_size` + +_[integer]_ - Number of concurrent connections and queries allowed + +### `allow_host_access` + +_[boolean]_ - Whether access to the local environment and file system is allowed + +### `cpu` + +_[integer]_ - Number of CPU cores available to the database + +### `memory_limit_gb` + +_[integer]_ - Amount of memory in GB available to the database + +### `read_write_ratio` + +_[number]_ - Ratio of resources allocated to the read database; used to divide CPU and memory + +### `init_sql` + +_[string]_ - is executed during database initialization. + +### `conn_init_sql` + +_[string]_ - is executed when a new connection is initialized. + +### `secrets` + +_[string]_ - Comma-separated list of other connector names to create temporary secrets for in DuckDB before executing a model. + +### `log_queries` + +_[boolean]_ - Whether to log raw SQL queries executed through OLAP + +#### Option 7: GCS + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the gcs + +## GCS + +Configuration properties specific to the gcs + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `gcs` _(required)_ + +### `google_application_credentials` + +_[string]_ - Google Cloud credentials JSON string + +### `bucket` + +_[string]_ - Name of gcs bucket _(required)_ + +### `allow_host_access` + +_[boolean]_ - Allow access to host environment configuration + +### `key_id` + +_[string]_ - Optional S3-compatible Key ID when used in compatibility mode + +### `secret` + +_[string]_ - Optional S3-compatible Secret when used in compatibility mode + +#### Option 8: HTTPS + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the https + +## HTTPS + +Configuration properties specific to the https + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `https` _(required)_ + +### `path` + +_[string]_ - The full HTTPS URI to fetch data from _(required)_ + +### `headers` + +_[object]_ - HTTP headers to include in the request + +#### Option 9: MotherDuck + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the motherduck + +## MotherDuck + +Configuration properties specific to the motherduck + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `duckdb` _(required)_ + +### `path` + +_[string]_ - Path to your MD database _(required)_ + +### `init_sql` + +_[string]_ - SQL executed during database initialization. _(required)_ + +#### Option 10: MySQL + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the mysql + +## MySQL + +Configuration properties specific to the mysql + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `mysql` _(required)_ + +### `dsn` + +_[string]_ - DSN(Data Source Name) for the mysql connection + +### `host` + +_[string]_ - Hostname of the MySQL server + +### `port` + +_[integer]_ - Port number for the MySQL server + +### `database` + +_[string]_ - Name of the MySQL database + +### `user` + +_[string]_ - Username for authentication + +### `password` + +_[string]_ - Password for authentication + +### `ssl_mode` + +_[string]_ - SSL mode can be DISABLED, PREFERRED or REQUIRED + +#### Option 11: Pinot + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the pinot + +## Pinot + +Configuration properties specific to the pinot + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `pinot` _(required)_ + +### `dsn` + +_[string]_ - DSN(Data Source Name) for the Pinot connection _(required)_ + +### `username` + +_[string]_ - Username for authenticating with Pinot + +### `password` + +_[string]_ - Password for authenticating with Pinot + +### `broker_host` + +_[string]_ - Hostname of the Pinot broker _(required)_ + +### `broker_port` + +_[integer]_ - Port number for the Pinot broker + +### `controller_host` + +_[string]_ - Hostname of the Pinot controller _(required)_ + +### `controller_port` + +_[integer]_ - Port number for the Pinot controller + +### `ssl` + +_[boolean]_ - Enable SSL connection to Pinot + +### `log_queries` + +_[boolean]_ - Log raw SQL queries executed through Pinot + +### `max_open_conns` + +_[integer]_ - Maximum number of open connections to the Pinot database + +#### Option 12: Postgres + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the postgres + +## Postgres + +Configuration properties specific to the postgres + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `postgres` _(required)_ + +### `dsn` + +_[string]_ - DSN(Data Source Name) for the postgres connection + +### `host` + +_[string]_ - Hostname of the Postgres server + +### `port` + +_[string]_ - Port number for the Postgres server + +### `dbname` + +_[string]_ - Name of the Postgres database + +### `user` + +_[string]_ - Username for authentication + +### `password` + +_[string]_ - Password for authentication + +### `sslmode` + +_[string]_ - SSL mode can be disable, allow, prefer or require + +#### Option 13: Redshift + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the redshift + +## Redshift + +Configuration properties specific to the redshift + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `redshift` _(required)_ + +### `aws_access_key_id` + +_[string]_ - AWS Access Key ID used for authenticating with Redshift. _(required)_ + +### `aws_secret_access_key` + +_[string]_ - AWS Secret Access Key used for authenticating with Redshift. _(required)_ + +### `aws_access_token` + +_[string]_ - AWS Session Token for temporary credentials (optional). + +### `region` + +_[string]_ - AWS region where the Redshift cluster or workgroup is hosted (e.g., 'us-east-1'). + +### `database` + +_[string]_ - Name of the Redshift database to query. _(required)_ + +### `workgroup` + +_[string]_ - Workgroup name for Redshift Serverless, in case of provisioned Redshift clusters use 'cluster_identifier'. + +### `cluster_identifier` + +_[string]_ - Cluster identifier for provisioned Redshift clusters, in case of Redshift Serverless use 'workgroup' . + +#### Option 14: S3 + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the s3 + +## S3 + +Configuration properties specific to the s3 + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `s3` _(required)_ + +### `aws_access_key_id` + +_[string]_ - AWS Access Key ID used for authentication + +### `aws_secret_access_key` + +_[string]_ - AWS Secret Access Key used for authentication + +### `aws_access_token` + +_[string]_ - Optional AWS session token for temporary credentials + +### `bucket` + +_[string]_ - Name of s3 bucket _(required)_ + +### `endpoint` + +_[string]_ - Optional custom endpoint URL for S3-compatible storage + +### `region` + +_[string]_ - AWS region of the S3 bucket + +### `allow_host_access` + +_[boolean]_ - Allow access to host environment configuration + +### `retain_files` + +_[boolean]_ - Whether to retain intermediate files after processing + +#### Option 15: Salesforce + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the salesforce + +## Salesforce + +Configuration properties specific to the salesforce + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `salesforce` _(required)_ + +### `username` + +_[string]_ - Salesforce account username _(required)_ + +### `password` + +_[string]_ - Salesforce account password (secret) + +### `key` + +_[string]_ - Authentication key for Salesforce (secret) + +### `endpoint` + +_[string]_ - Salesforce API endpoint URL _(required)_ + +### `client_id` + +_[string]_ - Client ID used for Salesforce OAuth authentication + +#### Option 16: Slack + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the slack + +## Slack + +Configuration properties specific to the slack + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `slack` _(required)_ + +### `bot_token` + +_[string]_ - Bot token used for authenticating Slack API requests _(required)_ + +#### Option 17: Snowflake + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the snowflake + +## Snowflake + +Configuration properties specific to the snowflake + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `snowflake` _(required)_ + +### `dsn` + +_[string]_ - DSN (Data Source Name) for the Snowflake connection _(required)_ + +### `parallel_fetch_limit` + +_[integer]_ - Maximum number of concurrent fetches during query execution + +#### Option 18: SQLite + +**Type:** _[object]_ + +**Description:** Configuration properties specific to the sqlite + +## SQLite + +Configuration properties specific to the sqlite + +### `driver` + +_[string]_ - Refers to the driver type and must be driver `sqlite` _(required)_ + +### `dsn` + +_[string]_ - DSN(Data Source Name) for the sqlite connection _(required)_ \ No newline at end of file diff --git a/docs/docs/hidden/yaml/explore-dashboards.md b/docs/docs/hidden/yaml/explore-dashboards.md new file mode 100644 index 00000000000..7361ec9ad44 --- /dev/null +++ b/docs/docs/hidden/yaml/explore-dashboards.md @@ -0,0 +1,295 @@ +--- +note: GENERATED. DO NOT EDIT. +title: Explore Dashboards YAML +sidebar_position: 35 +--- + +In your Rill project directory, create a explore dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. + +## Properties + +### `type` + +_[string]_ - Refers to the resource type and must be `explore` _(required)_ + +### `display_name` + +_[string]_ - Refers to the display name for the explore dashboard + +### `description` + +_[string]_ - Refers to the description of the explore dashboard + +### `banner` + +_[string]_ - Refers to the custom banner displayed at the header of an explore dashboard + +### `metrics_view` + +_[string]_ - Refers to the metrics view resource + +### `dimensions` + +_[oneOf]_ - List of dimension names. Use '*' to select all dimensions (default) + +#### Option 1 + +**Type:** _[string]_ + +**Description:** Wildcard(*) selector that includes all available fields in the selection + +#### Option 2 + +**Type:** _[array of string]_ + +**Description:** Explicit list of fields to include in the selection + +#### Option 3 + +**Type:** _[object]_ + +**Description:** Advanced matching using regex, DuckDB expression, or exclusion + + - **`regex`** - _[string]_ - Select dimensions using a regular expression + + - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic + + - **`exclude`** - _[object]_ - Select all dimensions except those listed here + +### `measures` + +_[oneOf]_ - List of measure names. Use ''*'' to select all measures (default) + +#### Option 1 + +**Type:** _[string]_ + +**Description:** Wildcard(*) selector that includes all available fields in the selection + +#### Option 2 + +**Type:** _[array of string]_ + +**Description:** Explicit list of fields to include in the selection + +#### Option 3 + +**Type:** _[object]_ + +**Description:** Advanced matching using regex, DuckDB expression, or exclusion + + - **`regex`** - _[string]_ - Select dimensions using a regular expression + + - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic + + - **`exclude`** - _[object]_ - Select all dimensions except those listed here + +### `theme` + +_[oneOf]_ - Name of the theme to use. Only one of theme and embedded_theme can be set. + +#### Option 1 + +**Type:** _[string]_ + +**Description:** Name of an existing theme to apply to the dashboard + +#### Option 2 + +**Type:** _[object]_ + +**Description:** Inline theme configuration. + + - **`colors`** - _[object]_ - Used to override the dashboard colors. Either primary or secondary color must be provided. + + - **`primary`** - _[string]_ - Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). + + - **`secondary`** - _[string]_ - Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. + +### `time_ranges` + +_[array of oneOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' + +#### Option 1 + +**Type:** _[string]_ + +**Description:** a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection + +#### Option 2 + +**Type:** _[object]_ + +**Description:** Object containing time range and comparison configuration + + - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ + + - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + +#### Option 1 + +**Type:** _[string]_ + +**Description:** Offset string only (range is inferred) + +#### Option 2 + +**Type:** _[object]_ + +**Description:** Object containing offset and range configuration for time comparison + + - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) + + - **`range`** - _[string]_ - Custom time range for comparison period + +### `time_zones` + +_[array of string]_ - Refers to the time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) + +### `lock_time_zone` + +_[boolean]_ - When true, the dashboard will be locked to the first time provided in the time_zones list. When no time_zones are provided, the dashboard will be locked to UTC + +### `allow_custom_time_range` + +_[boolean]_ - Defaults to true, when set to false it will hide the ability to set a custom time range for the user. + +### `defaults` + +_[object]_ - defines the defaults YAML struct + + - **`dimensions`** - _[oneOf]_ - Provides the default dimensions to load on viewing the dashboard + +#### Option 1 + +**Type:** _[string]_ + +**Description:** Wildcard(*) selector that includes all available fields in the selection + +#### Option 2 + +**Type:** _[array of string]_ + +**Description:** Explicit list of fields to include in the selection + +#### Option 3 + +**Type:** _[object]_ + +**Description:** Advanced matching using regex, DuckDB expression, or exclusion + + - **`regex`** - _[string]_ - Select dimensions using a regular expression + + - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic + + - **`exclude`** - _[object]_ - Select all dimensions except those listed here + + - **`measures`** - _[oneOf]_ - Provides the default measures to load on viewing the dashboard + +#### Option 1 + +**Type:** _[string]_ + +**Description:** Wildcard(*) selector that includes all available fields in the selection + +#### Option 2 + +**Type:** _[array of string]_ + +**Description:** Explicit list of fields to include in the selection + +#### Option 3 + +**Type:** _[object]_ + +**Description:** Advanced matching using regex, DuckDB expression, or exclusion + + - **`regex`** - _[string]_ - Select dimensions using a regular expression + + - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic + + - **`exclude`** - _[object]_ - Select all dimensions except those listed here + + - **`time_range`** - _[string]_ - Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + + - **`comparison_mode`** - _[string]_ - Controls how to compare current data with historical or categorical baselines. Options: `none` (no comparison), `time` (compares with past based on default_time_range), `dimension` (compares based on comparison_dimension values) + + - **`comparison_dimension`** - _[string]_ - for dimension mode, specify the comparison dimension by name + +### `embeds` + +_[object]_ - Configuration options for embedded dashboard views + + - **`hide_pivot`** - _[boolean]_ - When true, hides the pivot table view in embedded mode + +### `security` + +_[object]_ - Defines security rules and access control policies for resources + + - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + +#### Option 1 + +**Type:** _[string]_ + +**Description:** SQL expression that evaluates to a boolean to determine access + +#### Option 2 + +**Type:** _[boolean]_ + +**Description:** Direct boolean value to allow or deny access + + - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause + + - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded + + - **`if`** - _[string]_ - Expression to decide if the column should be included or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ + + - **`names`** - _[anyOf]_ - List of fields to include. Should match the name of one of the dashboard's dimensions or measures _(required)_ + + - **option 1** - _[array of string]_ - List of specific field names to include + + - **option 2** - _[string]_ - Wildcard '*' to include all fields + + - **`exclude`** - _[array of object]_ - List of dimension or measure names to exclude from the dashboard. If exclude is defined all other dimensions and measures are included + + - **`if`** - _[string]_ - Expression to decide if the column should be excluded or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ + + - **`names`** - _[anyOf]_ - List of fields to exclude. Should match the name of one of the dashboard's dimensions or measures _(required)_ + + - **option 1** - _[array of string]_ - List of specific field names to exclude + + - **option 2** - _[string]_ - Wildcard '*' to exclude all fields + + - **`rules`** - _[array of object]_ - List of detailed security rules that can be used to define complex access control policies + + - **`type`** - _[string]_ - Type of security rule - access (overall access), field_access (field-level access), or row_filter (row-level filtering) _(required)_ + + - **`action`** - _[string]_ - Whether to allow or deny access for this rule + + - **`if`** - _[string]_ - Conditional expression that determines when this rule applies. Must be a valid SQL expression that evaluates to a boolean + + - **`names`** - _[array of string]_ - List of field names this rule applies to (for field_access type rules) + + - **`all`** - _[boolean]_ - When true, applies the rule to all fields (for field_access type rules) + + - **`sql`** - _[string]_ - SQL expression for row filtering (for row_filter type rules) + +## Common Properties + +### `name` + +_[string]_ - Name is usually inferred from the filename, but can be specified manually. + +### `refs` + +_[array of string]_ - List of resource references + +### `dev` + +_[object]_ - Overrides any properties in development environment. + +### `prod` + +_[object]_ - Overrides any properties in production environment. \ No newline at end of file diff --git a/docs/docs/hidden/yaml/index.md b/docs/docs/hidden/yaml/index.md index 60f1d490f70..bbe5fa94aec 100644 --- a/docs/docs/hidden/yaml/index.md +++ b/docs/docs/hidden/yaml/index.md @@ -10,7 +10,7 @@ When you create models and dashboards, these objects are represented as object f :::info Working with resources outside their native folders -It is possible to define resources (such as [models](model.md), [metrics-views](metrics-view.md), [dashboards](explore.md), [custom APIs](api.md), or [themes](theme.md)) within any nested folder within your Rill project directory. However, for any YAML configuration file, it is then imperative that the `type` property is then appropriately defined within the underlying resource configuration or Rill will not able to resolve the resource type correctly! +It is possible to define resources (such as [models](advanced-models.md), [metrics-views](metrics-views.md), [dashboards](explore-dashboards.md), [custom APIs](apis.md), or [themes](themes.md)) within any nested folder within your Rill project directory. However, for any YAML configuration file, it is then imperative that the `type` property is then appropriately defined within the underlying resource configuration or Rill will not able to resolve the resource type correctly! ::: @@ -28,13 +28,12 @@ For more information about using Git or cloning projects locally, please see our ## Project files types -- [Alert YAML](alert.md) -- [API YAML](api.md) -- [Canvas YAML](canvas.md) -- [Component YAML](component.md) -- [Connector YAML](connector.md) -- [Explore YAML](explore.md) -- [Metrics View YAML](metrics-view.md) -- [Model YAML](model.md) -- [Theme YAML](theme.md) +- [Alerts YAML](alerts.md) +- [APIs YAML](apis.md) +- [Canvas Dashboards YAML](canvas-dashboards.md) +- [Connectors YAML](connectors.md) +- [Explore Dashboards YAML](explore-dashboards.md) +- [Metrics Views YAML](metrics-views.md) +- [Advanced Models YAML](advanced-models.md) +- [Themes YAML](themes.md) - [Project YAML](project.md) \ No newline at end of file diff --git a/docs/docs/hidden/yaml/metrics-views.md b/docs/docs/hidden/yaml/metrics-views.md new file mode 100644 index 00000000000..61730e64d50 --- /dev/null +++ b/docs/docs/hidden/yaml/metrics-views.md @@ -0,0 +1,186 @@ +--- +note: GENERATED. DO NOT EDIT. +title: Metrics Views YAML +sidebar_position: 36 +--- + +In your Rill project directory, create a metrics view, `.yaml`, file in the `metrics` directory. Rill will ingest the metric view definition next time you run `rill start`. + +## Properties + +### `type` + +_[string]_ - Refers to the resource type and must be `metrics_view` + +### `display_name` + +_[string]_ - Refers to the display name for the metrics view + +### `description` + +_[string]_ - Refers to the description for the metrics view + +### `ai_instructions` + +_[string]_ - Extra instructions for AI agents. Used to guide natural language question answering and routing. + +### `model` + +_[string]_ - Refers to the model powering the dashboard (either model or table is required) + +### `database` + +_[string]_ - Refers to the database to use in the OLAP engine (to be used in conjunction with table). Otherwise, will use the default database or schema if not specified + +### `database_schema` + +_[string]_ - Refers to the schema to use in the OLAP engine (to be used in conjunction with table). Otherwise, will use the default database or schema if not specified + +### `table` + +_[string]_ - Refers to the table powering the dashboard, should be used instead of model for dashboards create from external OLAP tables (either table or model is required) + +### `timeseries` + +_[string]_ - Refers to the timestamp column from your model that will underlie x-axis data in the line charts. If not specified, the line charts will not appear + +### `watermark` + +_[string]_ - A SQL expression that tells us the max timestamp that the metrics are considered valid for. Usually does not need to be overwritten + +### `smallest_time_grain` + +_[string]_ - Refers to the smallest time granularity the user is allowed to view. The valid values are: millisecond, second, minute, hour, day, week, month, quarter, year + +### `first_day_of_week` + +_[integer]_ - Refers to the first day of the week for time grain aggregation (for example, Sunday instead of Monday). The valid values are 1 through 7 where Monday=1 and Sunday=7 + +### `first_month_of_year` + +_[integer]_ - Refers to the first month of the year for time grain aggregation. The valid values are 1 through 12 where January=1 and December=12 + +### `dimensions` + +_[array of object]_ - Relates to exploring segments or dimensions of your data and filtering the dashboard + + - **`name`** - _[string]_ - a stable identifier for the dimension + + - **`display_name`** - _[string]_ - a display name for your dimension + + - **`description`** - _[string]_ - a freeform text description of the dimension + + - **`column`** - _[string]_ - a categorical column + + - **`expression`** - _[string]_ - a non-aggregate expression such as string_split(domain, '.'). One of column and expression is required but cannot have both at the same time + + - **`unnest`** - _[boolean]_ - if true, allows multi-valued dimension to be unnested (such as lists) and filters will automatically switch to "contains" instead of exact match + + - **`uri`** - _[string, boolean]_ - enable if your dimension is a clickable URL to enable single click navigation (boolean or valid SQL expression) + +### `measures` + +_[array of object]_ - Used to define the numeric aggregates of columns from your data model + + - **`name`** - _[string]_ - a stable identifier for the measure + + - **`display_name`** - _[string]_ - the display name of your measure. + + - **`description`** - _[string]_ - a freeform text description of the dimension + + - **`type`** - _[string]_ - Measure calculation type: "simple" for basic aggregations, "derived" for calculations using other measures, or "time_comparison" for period-over-period analysis. Defaults to "simple" unless dependencies exist. + + - **`expression`** - _[string]_ - a combination of operators and functions for aggregations + + - **`window`** - _[anyOf]_ - A measure window can be defined as a keyword string (e.g. 'time' or 'all') or an object with detailed window configuration. + + - **option 1** - _[string]_ - Shorthand: `time` or `true` means time-partitioned, `all` means non-partitioned. + + - **option 2** - _[object]_ - Detailed window configuration for measure calculations, allowing control over partitioning, ordering, and frame definition. + + - **`partition`** - _[boolean]_ - Controls whether the window is partitioned. When true, calculations are performed within each partition separately. + + - **`order`** - _[anyOf]_ - Specifies the fields to order the window by, determining the sequence of rows within each partition. + + - **option 1** - _[string]_ - Simple field name as a string. + + - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. + + - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. + + - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. + + - **`name`** - _[string]_ - Name of the field to select. _(required)_ + + - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. + + - **`frame`** - _[string]_ - Defines the window frame boundaries for calculations, specifying which rows are included in the window relative to the current row. + + - **`per`** - _[anyOf]_ - for per dimensions + + - **option 1** - _[string]_ - Simple field name as a string. + + - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. + + - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. + + - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. + + - **`name`** - _[string]_ - Name of the field to select. _(required)_ + + - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. + + - **`requires`** - _[anyOf]_ - using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures + + - **option 1** - _[string]_ - Simple field name as a string. + + - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. + + - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. + + - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. + + - **`name`** - _[string]_ - Name of the field to select. _(required)_ + + - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. + + - **`format_preset`** - _[string]_ - Controls the formatting of this measure using a predefined preset. Measures cannot have both `format_preset` and `format_d3`. If neither is supplied, the measure will be formatted using the `humanize` preset by default. + + Available options: + - `humanize`: Round numbers into thousands (K), millions(M), billions (B), etc. + - `none`: Raw output. + - `currency_usd`: Round to 2 decimal points with a dollar sign ($). + - `currency_eur`: Round to 2 decimal points with a euro sign (€). + - `percentage`: Convert a rate into a percentage with a % sign. + - `interval_ms`: Convert milliseconds into human-readable durations like hours (h), days (d), years (y), etc. (optional) + + + - **`format_d3`** - _[string]_ - Controls the formatting of this measure using a [d3-format](https://d3js.org/d3-format) string. If an invalid format string is supplied, the measure will fall back to `format_preset: humanize`. A measure cannot have both `format_preset` and `format_d3`. If neither is provided, the humanize preset is used by default. Example: `format_d3: ".2f"` formats using fixed-point notation with two decimal places. Example: `format_d3: ",.2r"` formats using grouped thousands with two significant digits. (optional) + + - **`format_d3_locale`** - _[object]_ - locale configuration passed through to D3, enabling changing the currency symbol among other things. For details, see the docs for D3's [formatLocale](https://d3js.org/d3-format#formatLocale) + + - **`valid_percent_of_total`** - _[boolean]_ - a boolean indicating whether percent-of-total values should be rendered for this measure + + - **`treat_nulls_as`** - _[string]_ - used to configure what value to fill in for missing time buckets. This also works generally as COALESCING over non empty time buckets. + +### `required` + +_[no type]_ - (no description) + +## Common Properties + +### `name` + +_[string]_ - Name is usually inferred from the filename, but can be specified manually. + +### `refs` + +_[array of string]_ - List of resource references + +### `dev` + +_[object]_ - Overrides any properties in development environment. + +### `prod` + +_[object]_ - Overrides any properties in production environment. \ No newline at end of file diff --git a/docs/docs/hidden/yaml/project.md b/docs/docs/hidden/yaml/project.md index 123702f4f09..8a7504e1b77 100644 --- a/docs/docs/hidden/yaml/project.md +++ b/docs/docs/hidden/yaml/project.md @@ -1,7 +1,7 @@ --- note: GENERATED. DO NOT EDIT. title: Project YAML -sidebar_position: 40 +sidebar_position: 39 --- The `rill.yaml` file contains metadata about your project. @@ -46,7 +46,7 @@ olap_connector: clickhouse ## Project-wide defaults -In `rill.yaml`, project-wide defaults can be specified for a resource type within a project. Unless otherwise specified, _individual resources will inherit any defaults_ that have been specified in `rill.yaml`. For available properties that can be configured, please refer to the YAML specification for each individual resource type - [model](model.md), [metrics_view](metrics-view.md), and [explore](explore.md) +In `rill.yaml`, project-wide defaults can be specified for a resource type within a project. Unless otherwise specified, _individual resources will inherit any defaults_ that have been specified in `rill.yaml`. For available properties that can be configured, please refer to the YAML specification for each individual resource type - [model](advanced-models.md), [metrics_view](metrics-views.md), and [explore](explore-dashboards.md) :::note Use plurals when specifying project-wide defaults In your `rill.yaml`, the top level property for the resource type needs to be **plural**, such as `models`, `metrics_views` and `explores`. @@ -54,7 +54,7 @@ In your `rill.yaml`, the top level property for the resource type needs to be ** :::info Hierarchy of inheritance and property overrides As a general rule of thumb, properties that have been specified at a more _granular_ level will supercede or override higher level properties that have been inherited. Therefore, in order of inheritance, Rill will prioritize properties in the following order: -1. Individual [models](model.md)/[metrics_views](metrics-view.md)/[explore](explore.md) object level properties (e.g. `model.yaml` or `explore.yaml`) +1. Individual [models](advanced-models.md)/[metrics_views](metrics-views.md)/[explore](explore-dashboards.md) object level properties (e.g. `advanced-models.yaml` or `explore-dashboards.yaml`) 2. [Environment](/docs/build/models/environments.md) level properties (e.g. a specific property that have been set for `dev`) 3. [Project-wide defaults](#project-wide-defaults) for a specific property and resource type ::: diff --git a/docs/docs/hidden/yaml/themes.md b/docs/docs/hidden/yaml/themes.md new file mode 100644 index 00000000000..35775e5b152 --- /dev/null +++ b/docs/docs/hidden/yaml/themes.md @@ -0,0 +1,52 @@ +--- +note: GENERATED. DO NOT EDIT. +title: Themes YAML +sidebar_position: 38 +--- + +In your Rill project directory, create a `.yaml` file in any directory containing `type: theme`. Rill will automatically ingest the theme next time you run `rill start` or deploy to Rill Cloud. + +To apply that theme to a dashboard, add `default_theme: ` to the yaml file for that dashboard. Alternatively, you can add this to the end of the URL in your browser: `?theme=` + + +## Properties + +### `type` + +_[string]_ - Refers to the resource type and must be `theme` _(required)_ + +### `colors` + +_[object]_ - Used to override the dashboard colors. Either primary or secondary color must be provided. _(required)_ + + - **`primary`** - _[string]_ - Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). + + - **`secondary`** - _[string]_ - Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. + +## Common Properties + +### `name` + +_[string]_ - Name is usually inferred from the filename, but can be specified manually. + +### `refs` + +_[array of string]_ - List of resource references + +### `dev` + +_[object]_ - Overrides any properties in development environment. + +### `prod` + +_[object]_ - Overrides any properties in production environment. + +## Examples + +```yaml +# Example: You can copy this directly into your .yaml file +type: theme +colors: + primary: plum + secondary: violet +``` \ No newline at end of file diff --git a/runtime/parser/schema/advanced-model.schema.yaml b/runtime/parser/schema/advanced-models.schema.yaml similarity index 96% rename from runtime/parser/schema/advanced-model.schema.yaml rename to runtime/parser/schema/advanced-models.schema.yaml index 448800d2839..d666994fc18 100644 --- a/runtime/parser/schema/advanced-model.schema.yaml +++ b/runtime/parser/schema/advanced-models.schema.yaml @@ -1,6 +1,6 @@ $schema: 'http://json-schema.org/draft-07/schema#' -$id: advanced-model.schema.yaml -title: Advanced Model YAML +$id: advanced-models.schema.yaml +title: Advanced Models YAML type: object allOf: - title: Properties @@ -14,7 +14,7 @@ allOf: $ref: '#/definitions/schedule_properties' description: Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying model data connector: - $ref: 'connector.schema.yaml#' + $ref: 'connectors.schema.yaml#' sql: type: string description: Raw SQL query to run against source @@ -150,7 +150,7 @@ allOf: - type: object allOf: - if: - title: Additional properties when `connector` is [`athena`](./connector#athena) + title: Additional properties when `connector` is [`athena`](./connectors#athena) properties: connector: const: athena @@ -159,7 +159,7 @@ allOf: then: $ref: '#/definitions/model/definitions/athena' - if: - title: Additional properties when `connector` is [`azure`](./connector#azure) + title: Additional properties when `connector` is [`azure`](./connectors#azure) properties: connector: const: azure @@ -168,7 +168,7 @@ allOf: then: $ref: '#/definitions/model/definitions/azure' - if: - title: Additional properties when `connector` is [`bigquery`](./connector#bigquery) + title: Additional properties when `connector` is [`bigquery`](./connectors#bigquery) properties: connector: const: bigquery @@ -177,7 +177,7 @@ allOf: then: $ref: '#/definitions/model/definitions/bigquery' - if: - title: Additional properties when `connector` is [`duckdb`](./connector#duckdb) + title: Additional properties when `connector` is [`duckdb`](./connectors#duckdb) properties: connector: const: duckdb @@ -186,7 +186,7 @@ allOf: then: $ref: '#/definitions/model/definitions/duckdb' - if: - title: Additional properties when `connector` is [`gcs`](./connector#gcs) + title: Additional properties when `connector` is [`gcs`](./connectors#gcs) properties: connector: const: gcs @@ -194,17 +194,17 @@ allOf: - connector then: $ref: '#/definitions/model/definitions/gcs' + # - if: + # title: Additional properties when `connector` is [`local_file`](./connectors#local_file) + # properties: + # connector: + # const: local_file + # required: + # - connector + # then: + # $ref: '#/definitions/model/definitions/local_file' - if: - title: Additional properties when `connector` is [`local_file`](./connector#local_file) - properties: - connector: - const: local_file - required: - - connector - then: - $ref: '#/definitions/model/definitions/local_file' - - if: - title: Additional properties when `connector` is [`redshift`](./connector#redshift) + title: Additional properties when `connector` is [`redshift`](./connectors#redshift) properties: connector: const: redshift @@ -213,7 +213,7 @@ allOf: then: $ref: '#/definitions/model/definitions/redshift' - if: - title: Additional properties when `connector` is [`s3`](./connector#s3) + title: Additional properties when `connector` is [`s3`](./connectors#s3) properties: connector: const: s3 diff --git a/runtime/parser/schema/alert.schema.yaml b/runtime/parser/schema/alerts.schema.yaml similarity index 99% rename from runtime/parser/schema/alert.schema.yaml rename to runtime/parser/schema/alerts.schema.yaml index 9efb755c557..5a7a80b5a25 100644 --- a/runtime/parser/schema/alert.schema.yaml +++ b/runtime/parser/schema/alerts.schema.yaml @@ -1,6 +1,6 @@ $schema: 'http://json-schema.org/draft-07/schema#' -$id: alert.schema.yaml -title: Alert YAML +$id: alerts.schema.yaml +title: Alerts YAML type: object description: Along with alertings at the dashboard level and can be created via the UI, there might be more extensive alerting that you might want to develop and can be done so the an alert.yaml. When creating an alert via a YAML file, you'll see this denoted in the UI as `Created through code`. examples: diff --git a/runtime/parser/schema/api.schema.yaml b/runtime/parser/schema/apis.schema.yaml similarity index 99% rename from runtime/parser/schema/api.schema.yaml rename to runtime/parser/schema/apis.schema.yaml index b7c1c39d3be..b978ef3f77e 100644 --- a/runtime/parser/schema/api.schema.yaml +++ b/runtime/parser/schema/apis.schema.yaml @@ -1,6 +1,6 @@ $schema: 'http://json-schema.org/draft-07/schema#' -$id: api.schema.yaml -title: API YAML +$id: apis.schema.yaml +title: APIs YAML type: object description: In your Rill project directory, create a new file name `.yaml` in the `apis` directory containing a custom API definition. See comprehensive documentation on how to define and use [custom APIs](/integrate/custom-apis/index.md) examples: diff --git a/runtime/parser/schema/canvas.schema.yaml b/runtime/parser/schema/canvas-dashboards.schema.yaml similarity index 97% rename from runtime/parser/schema/canvas.schema.yaml rename to runtime/parser/schema/canvas-dashboards.schema.yaml index e5b03211016..523d612f71f 100644 --- a/runtime/parser/schema/canvas.schema.yaml +++ b/runtime/parser/schema/canvas-dashboards.schema.yaml @@ -1,8 +1,8 @@ $schema: 'http://json-schema.org/draft-07/schema#' -$id: canvas.schema.yaml -title: Canvas YAML +$id: canvas-dashboards.schema.yaml +title: Canvas Dashboards YAML type: object -description: In your Rill project directory, create a explore dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. +description: In your Rill project directory, create a canvas dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. allOf: - title: Properties type: object diff --git a/runtime/parser/schema/connector.schema.yaml b/runtime/parser/schema/connectors.schema.yaml similarity index 99% rename from runtime/parser/schema/connector.schema.yaml rename to runtime/parser/schema/connectors.schema.yaml index 12fe1826fb8..2372acbc029 100644 --- a/runtime/parser/schema/connector.schema.yaml +++ b/runtime/parser/schema/connectors.schema.yaml @@ -1,6 +1,6 @@ $schema: 'http://json-schema.org/draft-07/schema#' -$id: connector.schema.yaml -title: Connector YAML +$id: connectors.schema.yaml +title: Connectors YAML type: object description: | Connector YAML files define how Rill connects to external data sources and OLAP engines. Each connector specifies a driver type and its required connection parameters. diff --git a/runtime/parser/schema/explore.schema.yaml b/runtime/parser/schema/explore-dashboards.schema.yaml similarity index 99% rename from runtime/parser/schema/explore.schema.yaml rename to runtime/parser/schema/explore-dashboards.schema.yaml index 88ba481b319..1eabb5cc500 100644 --- a/runtime/parser/schema/explore.schema.yaml +++ b/runtime/parser/schema/explore-dashboards.schema.yaml @@ -1,6 +1,6 @@ $schema: 'http://json-schema.org/draft-07/schema#' -$id: explore.schema.yaml -title: Explore YAML +$id: explore-dashboards.schema.yaml +title: Explore Dashboards YAML type: object description: In your Rill project directory, create a explore dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. allOf: diff --git a/runtime/parser/schema/metrics_view.schema.yaml b/runtime/parser/schema/metrics_views.schema.yaml similarity index 99% rename from runtime/parser/schema/metrics_view.schema.yaml rename to runtime/parser/schema/metrics_views.schema.yaml index 066351ea57a..b182c4b51f5 100644 --- a/runtime/parser/schema/metrics_view.schema.yaml +++ b/runtime/parser/schema/metrics_views.schema.yaml @@ -1,6 +1,6 @@ $schema: 'http://json-schema.org/draft-07/schema#' -$id: metrics_view.schema.yaml -title: Metrics View YAML +$id: metrics_views.schema.yaml +title: Metrics Views YAML type: object description: In your Rill project directory, create a metrics view, `.yaml`, file in the `metrics` directory. Rill will ingest the metric view definition next time you run `rill start`. allOf: diff --git a/runtime/parser/schema/project.schema.yaml b/runtime/parser/schema/project.schema.yaml index eca03218469..debcbc6203e 100644 --- a/runtime/parser/schema/project.schema.yaml +++ b/runtime/parser/schema/project.schema.yaml @@ -7,7 +7,7 @@ description: | :::info Working with resources outside their native folders - It is possible to define resources (such as [models](advanced-model.md), [metrics-views](metrics-view.md), [dashboards](explore.md), [custom APIs](api.md), or [themes](theme.md)) within any nested folder within your Rill project directory. However, for any YAML configuration file, it is then imperative that the `type` property is then appropriately defined within the underlying resource configuration or Rill will not able to resolve the resource type correctly! + It is possible to define resources (such as [models](advanced-models.md), [metrics-views](metrics-views.md), [dashboards](explore-dashboards.md), [custom APIs](apis.md), or [themes](themes.md)) within any nested folder within your Rill project directory. However, for any YAML configuration file, it is then imperative that the `type` property is then appropriately defined within the underlying resource configuration or Rill will not able to resolve the resource type correctly! ::: @@ -21,15 +21,15 @@ description: | ::: oneOf: - - $ref: 'alert.schema.yaml#' - - $ref: 'api.schema.yaml#' - - $ref: 'canvas.schema.yaml#' + - $ref: 'alerts.schema.yaml#' + - $ref: 'apis.schema.yaml#' + - $ref: 'canvas-dashboards.schema.yaml#' # - $ref: 'component.schema.yaml#' - - $ref: 'connector.schema.yaml#' - - $ref: 'explore.schema.yaml#' - - $ref: 'metrics_view.schema.yaml#' - - $ref: 'advanced-model.schema.yaml#' - - $ref: 'theme.schema.yaml#' + - $ref: 'connectors.schema.yaml#' + - $ref: 'explore-dashboards.schema.yaml#' + - $ref: 'metrics_views.schema.yaml#' + - $ref: 'advanced-models.schema.yaml#' + - $ref: 'themes.schema.yaml#' definitions: common_properties: type: object diff --git a/runtime/parser/schema/rillyaml.schema.yaml b/runtime/parser/schema/rillyaml.schema.yaml index 599a0d7f651..40d88195af2 100644 --- a/runtime/parser/schema/rillyaml.schema.yaml +++ b/runtime/parser/schema/rillyaml.schema.yaml @@ -38,7 +38,7 @@ allOf: - title: Project-wide defaults type: object description: | - In `rill.yaml`, project-wide defaults can be specified for a resource type within a project. Unless otherwise specified, _individual resources will inherit any defaults_ that have been specified in `rill.yaml`. For available properties that can be configured, please refer to the YAML specification for each individual resource type - [model]advanced-model.md), [metrics_view](metrics-view.md), and [explore](explore.md) + In `rill.yaml`, project-wide defaults can be specified for a resource type within a project. Unless otherwise specified, _individual resources will inherit any defaults_ that have been specified in `rill.yaml`. For available properties that can be configured, please refer to the YAML specification for each individual resource type - [model](advanced-models.md), [metrics_view](metrics-views.md), and [explore](explore-dashboards.md) :::note Use plurals when specifying project-wide defaults In your `rill.yaml`, the top level property for the resource type needs to be **plural**, such as `models`, `metrics_views` and `explores`. @@ -46,7 +46,7 @@ allOf: :::info Hierarchy of inheritance and property overrides As a general rule of thumb, properties that have been specified at a more _granular_ level will supercede or override higher level properties that have been inherited. Therefore, in order of inheritance, Rill will prioritize properties in the following order: - 1. Individual [models](advanced-model.md)/[metrics_views](metrics-view.md)/[explore](explore.md) object level properties (e.g. `model.yaml` or `explore.yaml`) + 1. Individual [models](advanced-models.md)/[metrics_views](metrics-views.md)/[explore](explore-dashboards.md) object level properties (e.g. `advanced-models.yaml` or `explore-dashboards.yaml`) 2. [Environment](/docs/build/models/environments.md) level properties (e.g. a specific property that have been set for `dev`) 3. [Project-wide defaults](#project-wide-defaults) for a specific property and resource type ::: diff --git a/runtime/parser/schema/theme.schema.yaml b/runtime/parser/schema/themes.schema.yaml similarity index 98% rename from runtime/parser/schema/theme.schema.yaml rename to runtime/parser/schema/themes.schema.yaml index 0151d4d201b..12e5d07d2b8 100644 --- a/runtime/parser/schema/theme.schema.yaml +++ b/runtime/parser/schema/themes.schema.yaml @@ -1,6 +1,6 @@ $schema: 'http://json-schema.org/draft-07/schema#' -$id: theme.schema.yaml -title: Theme YAML +$id: themes.schema.yaml +title: Themes YAML type: object description: | In your Rill project directory, create a `.yaml` file in any directory containing `type: theme`. Rill will automatically ingest the theme next time you run `rill start` or deploy to Rill Cloud. From 16c99c6d8b9a431031aa567228a2f826b21eb6a3 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 4 Aug 2025 16:33:48 -0600 Subject: [PATCH 12/32] reordered files and added sources YAML and model SQL --- cli/cmd/docs/generate_project.go | 12 +- docs/docs/hidden/yaml/advanced-models.md | 374 +--------- docs/docs/hidden/yaml/alert.md | 185 ----- docs/docs/hidden/yaml/alerts.md | 4 +- docs/docs/hidden/yaml/api.md | 154 ---- docs/docs/hidden/yaml/apis.md | 4 +- docs/docs/hidden/yaml/canvas-dashboards.md | 4 +- docs/docs/hidden/yaml/canvas.md | 177 ----- docs/docs/hidden/yaml/component.md | 59 -- docs/docs/hidden/yaml/connector.md | 671 ----------------- docs/docs/hidden/yaml/connectors.md | 674 +++++++++++------- docs/docs/hidden/yaml/explore-dashboards.md | 4 +- docs/docs/hidden/yaml/explore.md | 215 ------ docs/docs/hidden/yaml/index.md | 20 +- docs/docs/hidden/yaml/metrics-views.md | 4 +- .../{metrics-view.md => metrics_views.md} | 52 +- docs/docs/hidden/yaml/model.md | 398 ----------- docs/docs/hidden/yaml/models.md | 52 ++ .../hidden/yaml/{project.md => rillyaml.md} | 2 +- docs/docs/hidden/yaml/sources.md | 95 +++ docs/docs/hidden/yaml/theme.md | 52 -- docs/docs/hidden/yaml/themes.md | 4 +- .../parser/schema/advanced-models.schema.yaml | 2 +- runtime/parser/schema/alerts.schema.yaml | 2 +- runtime/parser/schema/apis.schema.yaml | 2 +- .../schema/canvas-dashboards.schema.yaml | 2 +- runtime/parser/schema/connectors.schema.yaml | 2 +- .../schema/explore-dashboards.schema.yaml | 2 +- .../parser/schema/metrics_views.schema.yaml | 4 +- runtime/parser/schema/models.schema.yaml | 54 ++ runtime/parser/schema/project.schema.yaml | 14 +- runtime/parser/schema/sources.schema.yaml | 100 +++ runtime/parser/schema/themes.schema.yaml | 2 +- 33 files changed, 790 insertions(+), 2612 deletions(-) delete mode 100644 docs/docs/hidden/yaml/alert.md delete mode 100644 docs/docs/hidden/yaml/api.md delete mode 100644 docs/docs/hidden/yaml/canvas.md delete mode 100644 docs/docs/hidden/yaml/component.md delete mode 100644 docs/docs/hidden/yaml/connector.md delete mode 100644 docs/docs/hidden/yaml/explore.md rename docs/docs/hidden/yaml/{metrics-view.md => metrics_views.md} (74%) delete mode 100644 docs/docs/hidden/yaml/model.md create mode 100644 docs/docs/hidden/yaml/models.md rename docs/docs/hidden/yaml/{project.md => rillyaml.md} (99%) create mode 100644 docs/docs/hidden/yaml/sources.md delete mode 100644 docs/docs/hidden/yaml/theme.md create mode 100644 runtime/parser/schema/models.schema.yaml create mode 100644 runtime/parser/schema/sources.schema.yaml diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index 1365a9b6a15..1e1072a8411 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -71,7 +71,17 @@ func GenerateProjectDocsCmd(rootCmd *cobra.Command, ch *cmdutil.Helper) *cobra.C requiredMap := getRequiredMapFromNode(resource) resourceFilebuf.WriteString(generateDoc(sidebarPosition, 0, resource, "", requiredMap)) resTitle := getScalarValue(resource, "title") - fileName := sanitizeFileName(resTitle) + ".md" + resId := getScalarValue(resource, "$id") + + // Use $id if available, otherwise fall back to title + var fileName string + if resId != "" { + // Remove .schema.yaml extension and convert to .md + fileName = strings.TrimSuffix(resId, ".schema.yaml") + ".md" + } else { + fileName = sanitizeFileName(resTitle) + ".md" + } + filePath := filepath.Join(outputDir, fileName) if err := os.WriteFile(filePath, []byte(resourceFilebuf.String()), 0o644); err != nil { return fmt.Errorf("failed writing resource doc: %w", err) diff --git a/docs/docs/hidden/yaml/advanced-models.md b/docs/docs/hidden/yaml/advanced-models.md index e8487e7740d..037e58bf701 100644 --- a/docs/docs/hidden/yaml/advanced-models.md +++ b/docs/docs/hidden/yaml/advanced-models.md @@ -1,7 +1,7 @@ --- note: GENERATED. DO NOT EDIT. -title: Advanced Models YAML -sidebar_position: 37 +title: Models YAML +sidebar_position: 34 --- ## Properties @@ -33,376 +33,6 @@ _[object]_ - Connector YAML files define how Rill connects to external data sour - **`driver`** - _[string]_ - The type of connector, see [available connectors](#available-connector-types) (required) _(required)_ -#### Option 1: Athena - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the athena - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `athena` _(required)_ - - - **`aws_access_key_id`** - _[string]_ - AWS Access Key ID used for authentication. Required when using static credentials directly or as base credentials for assuming a role. - - - **`aws_secret_access_key`** - _[string]_ - AWS Secret Access Key paired with the Access Key ID. Required when using static credentials directly or as base credentials for assuming a role. - - - **`aws_access_token`** - _[string]_ - AWS session token used with temporary credentials. Required only if the Access Key and Secret Key are part of a temporary session credentials. - - - **`role_arn`** - _[string]_ - ARN of the IAM role to assume. When specified, the SDK uses the base credentials to call STS AssumeRole and obtain temporary credentials scoped to this role. - - - **`role_session_name`** - _[string]_ - Session name to associate with the STS AssumeRole session. Used only if 'role_arn' is specified. Useful for identifying and auditing the session. - - - **`external_id`** - _[string]_ - External ID required by some roles when assuming them, typically for cross-account access. Used only if 'role_arn' is specified and the role's trust policy requires it. - - - **`workgroup`** - _[string]_ - Athena workgroup to use for query execution. Defaults to 'primary' if not specified. - - - **`output_location`** - _[string]_ - S3 URI where Athena query results should be stored (e.g., s3://your-bucket/athena/results/). Optional if the selected workgroup has a default result configuration. - - - **`aws_region`** - _[string]_ - AWS region where Athena and the result S3 bucket are located (e.g., us-east-1). Defaults to 'us-east-1' if not specified. - - - **`allow_host_access`** - _[boolean]_ - Allow the Athena client to access host environment configurations such as environment variables or local AWS credential files. Defaults to true, enabling use of credentials and settings from the host environment unless explicitly disabled. - -#### Option 2: Azure - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the azure - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `azure` _(required)_ - - - **`azure_storage_account`** - _[string]_ - Azure storage account name - - - **`azure_storage_key`** - _[string]_ - Azure storage access key - - - **`azure_storage_sas_token`** - _[string]_ - Optional azure SAS token for authentication - - - **`azure_storage_connection_string`** - _[string]_ - Optional azure connection string for storage account - - - **`azure_storage_bucket`** - _[string]_ - Name of the Azure Blob Storage container (equivalent to an S3 bucket) _(required)_ - - - **`allow_host_access`** - _[boolean]_ - Allow access to host environment configuration - -#### Option 3: BigQuery - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the bigquery - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `bigquery` _(required)_ - - - **`google_application_credentials`** - _[string]_ - Raw contents of the Google Cloud service account key (in JSON format) used for authentication. - - - **`project_id`** - _[string]_ - ID of the Google Cloud project to use for BigQuery operations. This can be omitted only if the project ID is included in the service account key. - - - **`allow_host_access`** - _[boolean]_ - Enable the BigQuery client to use credentials from the host environment when no service account JSON is provided. This includes Application Default Credentials from environment variables, local credential files, or the Google Compute Engine metadata server. Defaults to true, allowing seamless authentication in GCP environments. - -#### Option 4: ClickHouse - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the clickhouse - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `clickhouse` _(required)_ - - - **`managed`** - _[boolean]_ - `true` means Rill will provision the connector using the default provisioner. `false` disables automatic provisioning. - - - **`mode`** - _[string]_ - `read` - Controls the operation mode for the ClickHouse connection. Defaults to 'read' for safe operation with external databases. Set to 'readwrite' to enable model creation and table mutations. Note: When 'managed: true', this is automatically set to 'readwrite'. - - - **`dsn`** - _[string]_ - DSN(Data Source Name) for the ClickHouse connection - - - **`username`** - _[string]_ - Username for authentication - - - **`password`** - _[string]_ - Password for authentication - - - **`host`** - _[string]_ - Host where the ClickHouse instance is running - - - **`port`** - _[integer]_ - Port where the ClickHouse instance is accessible - - - **`database`** - _[string]_ - Name of the ClickHouse database within the cluster - - - **`ssl`** - _[boolean]_ - Indicates whether a secured SSL connection is required - - - **`cluster`** - _[string]_ - Cluster name, required for running distributed queries - - - **`log_queries`** - _[boolean]_ - Controls whether to log raw SQL queries - - - **`settings_override`** - _[string]_ - override the default settings used in queries. example `readonly = 1, session_timezone = 'UTC'` - - - **`embed_port`** - _[integer]_ - Port to run ClickHouse locally (0 for random port) - - - **`can_scale_to_zero`** - _[boolean]_ - Indicates if the database can scale to zero - - - **`max_open_conns`** - _[integer]_ - Maximum number of open connections to the database - - - **`max_idle_conns`** - _[integer]_ - Maximum number of idle connections in the pool - - - **`dial_timeout`** - _[string]_ - Timeout for dialing the ClickHouse server - - - **`conn_max_lifetime`** - _[string]_ - Maximum time a connection may be reused - - - **`read_timeout`** - _[string]_ - Maximum time for a connection to read data - -#### Option 5: Druid - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the druid - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `druid` _(required)_ - - - **`dsn`** - _[string]_ - Data Source Name (DSN) for connecting to Druid _(required)_ - - - **`username`** - _[string]_ - Username for authenticating with Druid - - - **`password`** - _[string]_ - Password for authenticating with Druid - - - **`host`** - _[string]_ - Hostname of the Druid coordinator or broker - - - **`port`** - _[integer]_ - Port number of the Druid service - - - **`ssl`** - _[boolean]_ - Enable SSL for secure connection - - - **`log_queries`** - _[boolean]_ - Log raw SQL queries sent to Druid - - - **`max_open_conns`** - _[integer]_ - Maximum number of open database connections (0 = default, -1 = unlimited) - - - **`skip_version_check`** - _[boolean]_ - Skip checking Druid version compatibility - -#### Option 6: DuckDB - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the duckdb - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `duckdb` _(required)_ - - - **`pool_size`** - _[integer]_ - Number of concurrent connections and queries allowed - - - **`allow_host_access`** - _[boolean]_ - Whether access to the local environment and file system is allowed - - - **`cpu`** - _[integer]_ - Number of CPU cores available to the database - - - **`memory_limit_gb`** - _[integer]_ - Amount of memory in GB available to the database - - - **`read_write_ratio`** - _[number]_ - Ratio of resources allocated to the read database; used to divide CPU and memory - - - **`init_sql`** - _[string]_ - is executed during database initialization. - - - **`conn_init_sql`** - _[string]_ - is executed when a new connection is initialized. - - - **`secrets`** - _[string]_ - Comma-separated list of other connector names to create temporary secrets for in DuckDB before executing a model. - - - **`log_queries`** - _[boolean]_ - Whether to log raw SQL queries executed through OLAP - -#### Option 7: GCS - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the gcs - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `gcs` _(required)_ - - - **`google_application_credentials`** - _[string]_ - Google Cloud credentials JSON string - - - **`bucket`** - _[string]_ - Name of gcs bucket _(required)_ - - - **`allow_host_access`** - _[boolean]_ - Allow access to host environment configuration - - - **`key_id`** - _[string]_ - Optional S3-compatible Key ID when used in compatibility mode - - - **`secret`** - _[string]_ - Optional S3-compatible Secret when used in compatibility mode - -#### Option 8: HTTPS - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the https - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `https` _(required)_ - - - **`path`** - _[string]_ - The full HTTPS URI to fetch data from _(required)_ - - - **`headers`** - _[object]_ - HTTP headers to include in the request - -#### Option 9: MotherDuck - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the motherduck - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `duckdb` _(required)_ - - - **`path`** - _[string]_ - Path to your MD database _(required)_ - - - **`init_sql`** - _[string]_ - SQL executed during database initialization. _(required)_ - -#### Option 10: MySQL - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the mysql - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `mysql` _(required)_ - - - **`dsn`** - _[string]_ - DSN(Data Source Name) for the mysql connection - - - **`host`** - _[string]_ - Hostname of the MySQL server - - - **`port`** - _[integer]_ - Port number for the MySQL server - - - **`database`** - _[string]_ - Name of the MySQL database - - - **`user`** - _[string]_ - Username for authentication - - - **`password`** - _[string]_ - Password for authentication - - - **`ssl_mode`** - _[string]_ - SSL mode can be DISABLED, PREFERRED or REQUIRED - -#### Option 11: Pinot - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the pinot - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `pinot` _(required)_ - - - **`dsn`** - _[string]_ - DSN(Data Source Name) for the Pinot connection _(required)_ - - - **`username`** - _[string]_ - Username for authenticating with Pinot - - - **`password`** - _[string]_ - Password for authenticating with Pinot - - - **`broker_host`** - _[string]_ - Hostname of the Pinot broker _(required)_ - - - **`broker_port`** - _[integer]_ - Port number for the Pinot broker - - - **`controller_host`** - _[string]_ - Hostname of the Pinot controller _(required)_ - - - **`controller_port`** - _[integer]_ - Port number for the Pinot controller - - - **`ssl`** - _[boolean]_ - Enable SSL connection to Pinot - - - **`log_queries`** - _[boolean]_ - Log raw SQL queries executed through Pinot - - - **`max_open_conns`** - _[integer]_ - Maximum number of open connections to the Pinot database - -#### Option 12: Postgres - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the postgres - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `postgres` _(required)_ - - - **`dsn`** - _[string]_ - DSN(Data Source Name) for the postgres connection - - - **`host`** - _[string]_ - Hostname of the Postgres server - - - **`port`** - _[string]_ - Port number for the Postgres server - - - **`dbname`** - _[string]_ - Name of the Postgres database - - - **`user`** - _[string]_ - Username for authentication - - - **`password`** - _[string]_ - Password for authentication - - - **`sslmode`** - _[string]_ - SSL mode can be disable, allow, prefer or require - -#### Option 13: Redshift - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the redshift - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `redshift` _(required)_ - - - **`aws_access_key_id`** - _[string]_ - AWS Access Key ID used for authenticating with Redshift. _(required)_ - - - **`aws_secret_access_key`** - _[string]_ - AWS Secret Access Key used for authenticating with Redshift. _(required)_ - - - **`aws_access_token`** - _[string]_ - AWS Session Token for temporary credentials (optional). - - - **`region`** - _[string]_ - AWS region where the Redshift cluster or workgroup is hosted (e.g., 'us-east-1'). - - - **`database`** - _[string]_ - Name of the Redshift database to query. _(required)_ - - - **`workgroup`** - _[string]_ - Workgroup name for Redshift Serverless, in case of provisioned Redshift clusters use 'cluster_identifier'. - - - **`cluster_identifier`** - _[string]_ - Cluster identifier for provisioned Redshift clusters, in case of Redshift Serverless use 'workgroup' . - -#### Option 14: S3 - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the s3 - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `s3` _(required)_ - - - **`aws_access_key_id`** - _[string]_ - AWS Access Key ID used for authentication - - - **`aws_secret_access_key`** - _[string]_ - AWS Secret Access Key used for authentication - - - **`aws_access_token`** - _[string]_ - Optional AWS session token for temporary credentials - - - **`bucket`** - _[string]_ - Name of s3 bucket _(required)_ - - - **`endpoint`** - _[string]_ - Optional custom endpoint URL for S3-compatible storage - - - **`region`** - _[string]_ - AWS region of the S3 bucket - - - **`allow_host_access`** - _[boolean]_ - Allow access to host environment configuration - - - **`retain_files`** - _[boolean]_ - Whether to retain intermediate files after processing - -#### Option 15: Salesforce - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the salesforce - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `salesforce` _(required)_ - - - **`username`** - _[string]_ - Salesforce account username _(required)_ - - - **`password`** - _[string]_ - Salesforce account password (secret) - - - **`key`** - _[string]_ - Authentication key for Salesforce (secret) - - - **`endpoint`** - _[string]_ - Salesforce API endpoint URL _(required)_ - - - **`client_id`** - _[string]_ - Client ID used for Salesforce OAuth authentication - -#### Option 16: Slack - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the slack - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `slack` _(required)_ - - - **`bot_token`** - _[string]_ - Bot token used for authenticating Slack API requests _(required)_ - -#### Option 17: Snowflake - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the snowflake - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `snowflake` _(required)_ - - - **`dsn`** - _[string]_ - DSN (Data Source Name) for the Snowflake connection _(required)_ - - - **`parallel_fetch_limit`** - _[integer]_ - Maximum number of concurrent fetches during query execution - -#### Option 18: SQLite - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the sqlite - - - **`driver`** - _[string]_ - Refers to the driver type and must be driver `sqlite` _(required)_ - - - **`dsn`** - _[string]_ - DSN(Data Source Name) for the sqlite connection _(required)_ - ### `sql` _[string]_ - Raw SQL query to run against source _(required)_ diff --git a/docs/docs/hidden/yaml/alert.md b/docs/docs/hidden/yaml/alert.md deleted file mode 100644 index 8f1c6aaa99d..00000000000 --- a/docs/docs/hidden/yaml/alert.md +++ /dev/null @@ -1,185 +0,0 @@ ---- -note: GENERATED. DO NOT EDIT. -title: Alert YAML -sidebar_position: 31 ---- - -Along with alertings at the dashboard level and can be created via the UI, there might be more extensive alerting that you might want to develop and can be done so the an alert.yaml. When creating an alert via a YAML file, you'll see this denoted in the UI as `Created through code`. - -## Properties - -### `type` - -_[string]_ - Refers to the resource type and must be `alert` _(required)_ - -### `display_name` - -_[string]_ - Refers to the display name for the alert - -### `refresh` - -_[object]_ - Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying data _(required)_ - - - **`cron`** - _[string]_ - A cron expression that defines the execution schedule - - - **`time_zone`** - _[string]_ - Time zone to interpret the schedule in (e.g., 'UTC', 'America/Los_Angeles'). - - - **`disable`** - _[boolean]_ - If true, disables the resource without deleting it. - - - **`ref_update`** - _[boolean]_ - If true, allows the resource to run when a dependency updates. - - - **`run_in_dev`** - _[boolean]_ - If true, allows the schedule to run in development mode. - -### `intervals` - -_[object]_ - define the interval of the alert to check - - - **`duration`** - _[string]_ - a valid ISO8601 duration to define the interval duration - - - **`limit`** - _[integer]_ - maximum number of intervals to check for on invocation - - - **`check_unclosed`** - _[boolean]_ - boolean, whether unclosed intervals should be checked - -### `watermark` - -_[string]_ - Specifies how the watermark is determined for incremental processing. Use 'trigger_time' to set it at runtime or 'inherit' to use the upstream model's watermark. - -### `timeout` - -_[string]_ - define the timeout of the alert in seconds (optional). - -### `data` - -_[oneOf]_ - Specifies one of the options to retrieve or compute the data used by alert _(required)_ - - - **option 1** - _[object]_ - Executes a raw SQL query against the project's data models. - - - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - - - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. - - - **option 2** - _[object]_ - Executes a SQL query that targets a defined metrics view. - - - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - - - **option 3** - _[object]_ - Calls a custom API defined in the project to compute data. - - - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - - - **`args`** - _[object]_ - Arguments to pass to the custom API. - - - **option 4** - _[object]_ - Uses a file-matching pattern (glob) to query data from a connector. - - - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - - - **option 1** - _[string]_ - A simple file path/glob pattern as a string. - - - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. - - - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. - - - **option 5** - _[object]_ - Uses the status of a resource as data. - - - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - - - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. - -### `for` - -_[oneOf]_ - Specifies how user identity or attributes should be evaluated for security policy enforcement. - - - **option 1** - _[object]_ - Specifies a unique user identifier for applying security policies. - - - **`user_id`** - _[string]_ - The unique user ID used to evaluate security policies. _(required)_ - - - **option 2** - _[object]_ - Specifies a user's email address for applying security policies. - - - **`user_email`** - _[string]_ - The user's email address used to evaluate security policies. _(required)_ - - - **option 3** - _[object]_ - Specifies a set of arbitrary user attributes for applying security policies. - - - **`attributes`** - _[object]_ - A dictionary of user attributes used to evaluate security policies. _(required)_ - -### `on_recover` - -_[boolean]_ - Send an alert when a previously failing alert recovers. Defaults to false. - -### `on_fail` - -_[boolean]_ - Send an alert when a failure occurs. Defaults to true. - -### `on_error` - -_[boolean]_ - Send an alert when an error occurs during evaluation. Defaults to false. - -### `renotify` - -_[boolean]_ - Enable repeated notifications for unresolved alerts. Defaults to false. - -### `renotify_after` - -_[string]_ - Defines the re-notification interval for the alert (e.g., '10m','24h'), equivalent to snooze duration in UI, defaults to 'Off' - -### `notify` - -_[object]_ - Defines how and where to send notifications. At least one method (email or Slack) is required. _(required)_ - - - **`email`** - _[object]_ - Send notifications via email. - - - **`recipients`** - _[array of string]_ - An array of email addresses to notify. _(required)_ - - - **`slack`** - _[object]_ - Send notifications via Slack. - - - **`users`** - _[array of string]_ - An array of Slack user IDs to notify. - - - **`channels`** - _[array of string]_ - An array of Slack channel IDs to notify. - - - **`webhooks`** - _[array of string]_ - An array of Slack webhook URLs to send notifications to. - -### `annotations` - -_[object]_ - Key value pair used for annotations - -## Common Properties - -### `name` - -_[string]_ - Name is usually inferred from the filename, but can be specified manually. - -### `refs` - -_[array of string]_ - List of resource references - -### `dev` - -_[object]_ - Overrides any properties in development environment. - -### `prod` - -_[object]_ - Overrides any properties in production environment. - -## Examples - -```yaml -# Example: To send alert when data lags by more than 1 day to slack channel #rill-cloud-alerts -type: alert -display_name: Data lags by more than 1 day -# Check the alert every hour. -refresh: - cron: 0 * * * * -# Query that returns non-empty results if the metrics lag by more than 1 day. -data: - sql: |- - SELECT * - FROM - ( - SELECT MAX(event_time) AS max_time - FROM rill_metrics_model - ) - WHERE max_time < NOW() - INTERVAL '1 day' -# Send notifications in Slack. -notify: - slack: - channels: - - '#rill-cloud-alerts' -``` \ No newline at end of file diff --git a/docs/docs/hidden/yaml/alerts.md b/docs/docs/hidden/yaml/alerts.md index f3719eded91..c405a8d7e95 100644 --- a/docs/docs/hidden/yaml/alerts.md +++ b/docs/docs/hidden/yaml/alerts.md @@ -1,7 +1,7 @@ --- note: GENERATED. DO NOT EDIT. -title: Alerts YAML -sidebar_position: 31 +title: Alert YAML +sidebar_position: 38 --- Along with alertings at the dashboard level and can be created via the UI, there might be more extensive alerting that you might want to develop and can be done so the an alert.yaml. When creating an alert via a YAML file, you'll see this denoted in the UI as `Created through code`. diff --git a/docs/docs/hidden/yaml/api.md b/docs/docs/hidden/yaml/api.md deleted file mode 100644 index 177e67c4243..00000000000 --- a/docs/docs/hidden/yaml/api.md +++ /dev/null @@ -1,154 +0,0 @@ ---- -note: GENERATED. DO NOT EDIT. -title: API YAML -sidebar_position: 32 ---- - -In your Rill project directory, create a new file name `.yaml` in the `apis` directory containing a custom API definition. See comprehensive documentation on how to define and use [custom APIs](/integrate/custom-apis/index.md) - -## Properties - -### `type` - -_[string]_ - Refers to the resource type and must be `api` _(required)_ - -### `openapi` - -_[object]_ - OpenAPI specification for the API endpoint - - - **`summary`** - _[string]_ - A brief description of what the API endpoint does - - - **`parameters`** - _[array of object]_ - List of parameters that the API endpoint accepts - - - **`request_schema`** - _[object]_ - JSON schema for the request body (use nested YAML instead of a JSON string) - - - **`response_schema`** - _[object]_ - JSON schema for the response body (use nested YAML instead of a JSON string) - -### `security` - -_[object]_ - Defines security rules and access control policies for resources - - - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. - - - **option 1** - _[string]_ - SQL expression that evaluates to a boolean to determine access - - - **option 2** - _[boolean]_ - Direct boolean value to allow or deny access - - - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause - - - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded - - - **`if`** - _[string]_ - Expression to decide if the column should be included or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ - - - **`names`** - _[anyOf]_ - List of fields to include. Should match the name of one of the dashboard's dimensions or measures _(required)_ - - - **option 1** - _[array of string]_ - List of specific field names to include - - - **option 2** - _[string]_ - Wildcard '*' to include all fields - - - **`exclude`** - _[array of object]_ - List of dimension or measure names to exclude from the dashboard. If exclude is defined all other dimensions and measures are included - - - **`if`** - _[string]_ - Expression to decide if the column should be excluded or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ - - - **`names`** - _[anyOf]_ - List of fields to exclude. Should match the name of one of the dashboard's dimensions or measures _(required)_ - - - **option 1** - _[array of string]_ - List of specific field names to exclude - - - **option 2** - _[string]_ - Wildcard '*' to exclude all fields - - - **`rules`** - _[array of object]_ - List of detailed security rules that can be used to define complex access control policies - - - **`type`** - _[string]_ - Type of security rule - access (overall access), field_access (field-level access), or row_filter (row-level filtering) _(required)_ - - - **`action`** - _[string]_ - Whether to allow or deny access for this rule - - - **`if`** - _[string]_ - Conditional expression that determines when this rule applies. Must be a valid SQL expression that evaluates to a boolean - - - **`names`** - _[array of string]_ - List of field names this rule applies to (for field_access type rules) - - - **`all`** - _[boolean]_ - When true, applies the rule to all fields (for field_access type rules) - - - **`sql`** - _[string]_ - SQL expression for row filtering (for row_filter type rules) - -### `skip_nested_security` - -_[boolean]_ - Flag to control security inheritance - -## One of Properties Options -- [SQL Query](#sql-query) -- [Metrics View Query](#metrics-view-query) -- [Custom API Call](#custom-api-call) -- [File Glob Query](#file-glob-query) -- [Resource Status Check](#resource-status-check) - -## SQL Query - -Executes a raw SQL query against the project's data models. - -### `sql` - -_[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - -### `connector` - -_[string]_ - specifies the connector to use when running SQL or glob queries. - -## Metrics View Query - -Executes a SQL query that targets a defined metrics view. - -### `metrics_sql` - -_[string]_ - SQL query that targets a metrics view in the project _(required)_ - -## Custom API Call - -Calls a custom API defined in the project to compute data. - -### `api` - -_[string]_ - Name of a custom API defined in the project. _(required)_ - -### `args` - -_[object]_ - Arguments to pass to the custom API. - -## File Glob Query - -Uses a file-matching pattern (glob) to query data from a connector. - -### `glob` - -_[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - - - **option 1** - _[string]_ - A simple file path/glob pattern as a string. - - - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. - -### `connector` - -_[string]_ - Specifies the connector to use with the glob input. - -## Resource Status Check - -Uses the status of a resource as data. - -### `resource_status` - -_[object]_ - Based on resource status _(required)_ - - - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. - -## Examples - -```yaml -# Example: This api returns the top 10 authors by net line changes since the specified date provided in the arguments. -type: api -name: metrics_view_api -metrics_sql: |- - SELECT author_name, net_line_changes - FROM advanced_metrics_view - where author_date > '{{ .args.date }}' - order by net_line_changes DESC - limit 10 -``` \ No newline at end of file diff --git a/docs/docs/hidden/yaml/apis.md b/docs/docs/hidden/yaml/apis.md index 729972b1117..00d49177c94 100644 --- a/docs/docs/hidden/yaml/apis.md +++ b/docs/docs/hidden/yaml/apis.md @@ -1,7 +1,7 @@ --- note: GENERATED. DO NOT EDIT. -title: APIs YAML -sidebar_position: 32 +title: API YAML +sidebar_position: 39 --- In your Rill project directory, create a new file name `.yaml` in the `apis` directory containing a custom API definition. See comprehensive documentation on how to define and use [custom APIs](/integrate/custom-apis/index.md) diff --git a/docs/docs/hidden/yaml/canvas-dashboards.md b/docs/docs/hidden/yaml/canvas-dashboards.md index 694b9a05730..68238dbdd44 100644 --- a/docs/docs/hidden/yaml/canvas-dashboards.md +++ b/docs/docs/hidden/yaml/canvas-dashboards.md @@ -1,7 +1,7 @@ --- note: GENERATED. DO NOT EDIT. -title: Canvas Dashboards YAML -sidebar_position: 33 +title: Canvas Dashboard YAML +sidebar_position: 36 --- In your Rill project directory, create a canvas dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. diff --git a/docs/docs/hidden/yaml/canvas.md b/docs/docs/hidden/yaml/canvas.md deleted file mode 100644 index 61a3c2d0cb8..00000000000 --- a/docs/docs/hidden/yaml/canvas.md +++ /dev/null @@ -1,177 +0,0 @@ ---- -note: GENERATED. DO NOT EDIT. -title: Canvas YAML -sidebar_position: 33 ---- - -In your Rill project directory, create a explore dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. - -## Properties - -### `type` - -_[string]_ - Refers to the resource type and must be `canvas` _(required)_ - -### `display_name` - -_[string]_ - Refers to the display name for the canvas - -### `banner` - -_[string]_ - Refers to the custom banner displayed at the header of an Canvas dashboard - -### `max_width` - -_[integer]_ - Max width in pixels of the canvas - -### `gap_x` - -_[integer]_ - Horizontal gap in pixels of the canvas - -### `gap_y` - -_[integer]_ - Vertical gap in pixels of the canvas - -### `theme` - -_[oneOf]_ - Name of the theme to use. Only one of theme and embedded_theme can be set. - - - **option 1** - _[string]_ - Name of an existing theme to apply to the dashboard - - - **option 2** - _[object]_ - Inline theme configuration. - - - **`colors`** - _[object]_ - Used to override the dashboard colors. Either primary or secondary color must be provided. - - - **`primary`** - _[string]_ - Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). - - - **`secondary`** - _[string]_ - Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. - -### `allow_custom_time_range` - -_[boolean]_ - Defaults to true, when set to false it will hide the ability to set a custom time range for the user. - -### `time_ranges` - -_[array of oneOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' - - - **option 1** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection - - - **option 2** - _[object]_ - Object containing time range and comparison configuration - - - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ - - - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) - - - **option 1** - _[string]_ - Offset string only (range is inferred) - - - **option 2** - _[object]_ - Object containing offset and range configuration for time comparison - - - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) - - - **`range`** - _[string]_ - Custom time range for comparison period - -### `time_zones` - -_[array of string]_ - Refers to the time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) - -### `filters` - -_[object]_ - Indicates if filters should be enabled for the canvas. - - - **`enable`** - _[boolean]_ - Toggles filtering functionality for the canvas dashboard. - -### `defaults` - -_[object]_ - Preset UI state to show by default - - - **`time_range`** - _[string]_ - Default time range to display when the dashboard loads - - - **`comparison_mode`** - _[string]_ - Default comparison mode for metrics (none, time, or dimension) - - - **`comparison_dimension`** - _[string]_ - Default dimension to use for comparison when comparison_mode is 'dimension' - -### `variables` - -_[array of object]_ - Variables that can be used in the canvas - - - **`name`** - _[string]_ - Unique identifier for the variable _(required)_ - - - **`type`** - _[string]_ - Data type of the variable (e.g., string, number, boolean) _(required)_ - - - **`value`** - _[string, number, boolean, object, array]_ - Default value for the variable. Can be any valid JSON value type - -### `rows` - -_[array of object]_ - Refers to all of the rows displayed on the Canvas _(required)_ - - - **`height`** - _[string]_ - Height of the row in px - - - **`items`** - _[array of object]_ - List of components to display in the row - - - **`component`** - _[string]_ - Name of the component to display - - - **`width`** - _[string, integer]_ - Width of the component (can be a number or string with unit) - -### `security` - -_[object]_ - Defines security rules and access control policies for resources - - - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. - - - **option 1** - _[string]_ - SQL expression that evaluates to a boolean to determine access - - - **option 2** - _[boolean]_ - Direct boolean value to allow or deny access - - - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause - - - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded - - - **`if`** - _[string]_ - Expression to decide if the column should be included or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ - - - **`names`** - _[anyOf]_ - List of fields to include. Should match the name of one of the dashboard's dimensions or measures _(required)_ - - - **option 1** - _[array of string]_ - List of specific field names to include - - - **option 2** - _[string]_ - Wildcard '*' to include all fields - - - **`exclude`** - _[array of object]_ - List of dimension or measure names to exclude from the dashboard. If exclude is defined all other dimensions and measures are included - - - **`if`** - _[string]_ - Expression to decide if the column should be excluded or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ - - - **`names`** - _[anyOf]_ - List of fields to exclude. Should match the name of one of the dashboard's dimensions or measures _(required)_ - - - **option 1** - _[array of string]_ - List of specific field names to exclude - - - **option 2** - _[string]_ - Wildcard '*' to exclude all fields - - - **`rules`** - _[array of object]_ - List of detailed security rules that can be used to define complex access control policies - - - **`type`** - _[string]_ - Type of security rule - access (overall access), field_access (field-level access), or row_filter (row-level filtering) _(required)_ - - - **`action`** - _[string]_ - Whether to allow or deny access for this rule - - - **`if`** - _[string]_ - Conditional expression that determines when this rule applies. Must be a valid SQL expression that evaluates to a boolean - - - **`names`** - _[array of string]_ - List of field names this rule applies to (for field_access type rules) - - - **`all`** - _[boolean]_ - When true, applies the rule to all fields (for field_access type rules) - - - **`sql`** - _[string]_ - SQL expression for row filtering (for row_filter type rules) - -## Common Properties - -### `name` - -_[string]_ - Name is usually inferred from the filename, but can be specified manually. - -### `refs` - -_[array of string]_ - List of resource references - -### `dev` - -_[object]_ - Overrides any properties in development environment. - -### `prod` - -_[object]_ - Overrides any properties in production environment. \ No newline at end of file diff --git a/docs/docs/hidden/yaml/component.md b/docs/docs/hidden/yaml/component.md deleted file mode 100644 index b969461ddfb..00000000000 --- a/docs/docs/hidden/yaml/component.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -note: GENERATED. DO NOT EDIT. -title: Component YAML -sidebar_position: 34 ---- - -Defines a reusable dashboard component that can be embedded in canvas dashboards - -## Properties - -### `type` - -_[string]_ - Refers to the resource type and must be `component` _(required)_ - -### `display_name` - -_[string]_ - Refers to the display name for the component - -### `description` - -_[string]_ - Detailed description of the component's purpose and functionality - -### `input` - -_[array of object]_ - List of input variables that can be passed to the component - - - **`name`** - _[string]_ - Unique identifier for the variable _(required)_ - - - **`type`** - _[string]_ - Data type of the variable (e.g., string, number, boolean) _(required)_ - - - **`value`** - _[string, number, boolean, object, array]_ - Default value for the variable. Can be any valid JSON value type - -### `output` - -_[object]_ - Output variable that the component produces - - - **`name`** - _[string]_ - Unique identifier for the variable _(required)_ - - - **`type`** - _[string]_ - Data type of the variable (e.g., string, number, boolean) _(required)_ - - - **`value`** - _[string, number, boolean, object, array]_ - Default value for the variable. Can be any valid JSON value type - -## Common Properties - -### `name` - -_[string]_ - Name is usually inferred from the filename, but can be specified manually. - -### `refs` - -_[array of string]_ - List of resource references - -### `dev` - -_[object]_ - Overrides any properties in development environment. - -### `prod` - -_[object]_ - Overrides any properties in production environment. \ No newline at end of file diff --git a/docs/docs/hidden/yaml/connector.md b/docs/docs/hidden/yaml/connector.md deleted file mode 100644 index 8bfa5ba1fdf..00000000000 --- a/docs/docs/hidden/yaml/connector.md +++ /dev/null @@ -1,671 +0,0 @@ ---- -note: GENERATED. DO NOT EDIT. -title: Connector YAML -sidebar_position: 35 ---- - -When you add olap_connector to your rill.yaml file, you will need to set up a `.yaml` file in the 'connectors' directory. This file requires the following parameters,type and driver (see below for more parameter options). Rill will automatically test the connectivity to the OLAP engine upon saving the file. This can be viewed in the connectors tab in the UI. - -:::tip Did you know? - -Starting from Rill 0.46, you can directly create OLAP engines from the UI! Select + Add -> Data -> Connect an OLAP engine - -::: - - -## Properties - -### `type` - -_[string]_ - Refers to the resource type and must be `connector` _(required)_ - -## Common Properties - -### `name` - -_[string]_ - Name is usually inferred from the filename, but can be specified manually. - -### `refs` - -_[array of string]_ - List of resource references - -### `dev` - -_[object]_ - Overrides any properties in development environment. - -### `prod` - -_[object]_ - Overrides any properties in production environment. - -## One of Properties Options -- [athena](#athena) -- [azure](#azure) -- [bigquery](#bigquery) -- [clickhouse](#clickhouse) -- [druid](#druid) -- [duckdb](#duckdb) -- [gcs](#gcs) -- [https](#https) -- [local_file](#local_file) -- [motherduck](#motherduck) -- [mysql](#mysql) -- [pinot](#pinot) -- [postgres](#postgres) -- [redshift](#redshift) -- [s3](#s3) -- [salesforce](#salesforce) -- [slack](#slack) -- [snowflake](#snowflake) -- [sqlite](#sqlite) - -## athena - -Configuration properties specific to the athena - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `athena` _(required)_ - -### `aws_access_key_id` - -_[string]_ - AWS Access Key ID used for authentication. Required when using static credentials directly or as base credentials for assuming a role. - -### `aws_secret_access_key` - -_[string]_ - AWS Secret Access Key paired with the Access Key ID. Required when using static credentials directly or as base credentials for assuming a role. - -### `aws_access_token` - -_[string]_ - AWS session token used with temporary credentials. Required only if the Access Key and Secret Key are part of a temporary session credentials. - -### `role_arn` - -_[string]_ - ARN of the IAM role to assume. When specified, the SDK uses the base credentials to call STS AssumeRole and obtain temporary credentials scoped to this role. - -### `role_session_name` - -_[string]_ - Session name to associate with the STS AssumeRole session. Used only if 'role_arn' is specified. Useful for identifying and auditing the session. - -### `external_id` - -_[string]_ - External ID required by some roles when assuming them, typically for cross-account access. Used only if 'role_arn' is specified and the role's trust policy requires it. - -### `workgroup` - -_[string]_ - Athena workgroup to use for query execution. Defaults to 'primary' if not specified. - -### `output_location` - -_[string]_ - S3 URI where Athena query results should be stored (e.g., s3://your-bucket/athena/results/). Optional if the selected workgroup has a default result configuration. - -### `aws_region` - -_[string]_ - AWS region where Athena and the result S3 bucket are located (e.g., us-east-1). Defaults to 'us-east-1' if not specified. - -### `allow_host_access` - -_[boolean]_ - Allow the Athena client to access host environment configurations such as environment variables or local AWS credential files. Defaults to true, enabling use of credentials and settings from the host environment unless explicitly disabled. - -## azure - -Configuration properties specific to the azure - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `azure` _(required)_ - -### `azure_storage_account` - -_[string]_ - Azure storage account name - -### `azure_storage_key` - -_[string]_ - Azure storage access key - -### `azure_storage_sas_token` - -_[string]_ - Optional azure SAS token for authentication - -### `azure_storage_connection_string` - -_[string]_ - Optional azure connection string for storage account - -### `azure_storage_bucket` - -_[string]_ - Name of the Azure Blob Storage container (equivalent to an S3 bucket) _(required)_ - -### `allow_host_access` - -_[boolean]_ - Allow access to host environment configuration - -## bigquery - -Configuration properties specific to the bigquery - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `bigquery` _(required)_ - -### `google_application_credentials` - -_[string]_ - Raw contents of the Google Cloud service account key (in JSON format) used for authentication. - -### `project_id` - -_[string]_ - ID of the Google Cloud project to use for BigQuery operations. This can be omitted only if the project ID is included in the service account key. - -### `allow_host_access` - -_[boolean]_ - Enable the BigQuery client to use credentials from the host environment when no service account JSON is provided. This includes Application Default Credentials from environment variables, local credential files, or the Google Compute Engine metadata server. Defaults to true, allowing seamless authentication in GCP environments. - -## clickhouse - -Configuration properties specific to the clickhouse - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `clickhouse` _(required)_ - -### `managed` - -_[boolean]_ - `true` means Rill will provision the connector using the default provisioner. `false` disables automatic provisioning. - -### `mode` - -_[string]_ - `read` - Controls the operation mode for the ClickHouse connection. Defaults to 'read' for safe operation with external databases. Set to 'readwrite' to enable model creation and table mutations. Note: When 'managed: true', this is automatically set to 'readwrite'. - -### `dsn` - -_[string]_ - DSN(Data Source Name) for the ClickHouse connection - -### `username` - -_[string]_ - Username for authentication - -### `password` - -_[string]_ - Password for authentication - -### `host` - -_[string]_ - Host where the ClickHouse instance is running - -### `port` - -_[integer]_ - Port where the ClickHouse instance is accessible - -### `database` - -_[string]_ - Name of the ClickHouse database within the cluster - -### `ssl` - -_[boolean]_ - Indicates whether a secured SSL connection is required - -### `cluster` - -_[string]_ - Cluster name, required for running distributed queries - -### `log_queries` - -_[boolean]_ - Controls whether to log raw SQL queries - -### `settings_override` - -_[string]_ - override the default settings used in queries. example `readonly = 1, session_timezone = 'UTC'` - -### `embed_port` - -_[integer]_ - Port to run ClickHouse locally (0 for random port) - -### `can_scale_to_zero` - -_[boolean]_ - Indicates if the database can scale to zero - -### `max_open_conns` - -_[integer]_ - Maximum number of open connections to the database - -### `max_idle_conns` - -_[integer]_ - Maximum number of idle connections in the pool - -### `dial_timeout` - -_[string]_ - Timeout for dialing the ClickHouse server - -### `conn_max_lifetime` - -_[string]_ - Maximum time a connection may be reused - -### `read_timeout` - -_[string]_ - Maximum time for a connection to read data - -## druid - -Configuration properties specific to the druid - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `druid` _(required)_ - -### `dsn` - -_[string]_ - Data Source Name (DSN) for connecting to Druid _(required)_ - -### `username` - -_[string]_ - Username for authenticating with Druid - -### `password` - -_[string]_ - Password for authenticating with Druid - -### `host` - -_[string]_ - Hostname of the Druid coordinator or broker - -### `port` - -_[integer]_ - Port number of the Druid service - -### `ssl` - -_[boolean]_ - Enable SSL for secure connection - -### `log_queries` - -_[boolean]_ - Log raw SQL queries sent to Druid - -### `max_open_conns` - -_[integer]_ - Maximum number of open database connections (0 = default, -1 = unlimited) - -### `skip_version_check` - -_[boolean]_ - Skip checking Druid version compatibility - -## duckdb - -Configuration properties specific to the duckdb - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `duckdb` _(required)_ - -### `pool_size` - -_[integer]_ - Number of concurrent connections and queries allowed - -### `allow_host_access` - -_[boolean]_ - Whether access to the local environment and file system is allowed - -### `cpu` - -_[integer]_ - Number of CPU cores available to the database - -### `memory_limit_gb` - -_[integer]_ - Amount of memory in GB available to the database - -### `read_write_ratio` - -_[number]_ - Ratio of resources allocated to the read database; used to divide CPU and memory - -### `init_sql` - -_[string]_ - is executed during database initialization. - -### `conn_init_sql` - -_[string]_ - is executed when a new connection is initialized. - -### `secrets` - -_[string]_ - Comma-separated list of other connector names to create temporary secrets for in DuckDB before executing a model. - -### `log_queries` - -_[boolean]_ - Whether to log raw SQL queries executed through OLAP - -## gcs - -Configuration properties specific to the gcs - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `gcs` _(required)_ - -### `google_application_credentials` - -_[string]_ - Google Cloud credentials JSON string - -### `bucket` - -_[string]_ - Name of gcs bucket _(required)_ - -### `allow_host_access` - -_[boolean]_ - Allow access to host environment configuration - -### `key_id` - -_[string]_ - Optional S3-compatible Key ID when used in compatibility mode - -### `secret` - -_[string]_ - Optional S3-compatible Secret when used in compatibility mode - -## https - -Configuration properties specific to the https - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `https` _(required)_ - -### `path` - -_[string]_ - The full HTTPS URI to fetch data from _(required)_ - -### `headers` - -_[object]_ - HTTP headers to include in the request - -## local_file - -Configuration properties specific to the local_file - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `local_file` _(required)_ - -### `dsn` - -_[string]_ - Data Source Name (DSN) indicating the file path or location of the local file _(required)_ - -### `allow_host_access` - -_[boolean]_ - Flag to indicate if access to host-level file paths is permitted - -## motherduck - -Configuration properties specific to the motherduck - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `motherduck` _(required)_ - -### `dsn` - -_[string]_ - Data Source Name (DSN) specifying the MotherDuck connection endpoint _(required)_ - -### `token` - -_[string]_ - Authentication token for accessing MotherDuck (secret) _(required)_ - -## mysql - -Configuration properties specific to the mysql - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `mysql` _(required)_ - -### `dsn` - -_[string]_ - DSN(Data Source Name) for the mysql connection - -### `host` - -_[string]_ - Hostname of the MySQL server - -### `port` - -_[integer]_ - Port number for the MySQL server - -### `database` - -_[string]_ - Name of the MySQL database - -### `user` - -_[string]_ - Username for authentication - -### `password` - -_[string]_ - Password for authentication - -### `ssl_mode` - -_[string]_ - SSL mode can be DISABLED, PREFERRED or REQUIRED - -## pinot - -Configuration properties specific to the pinot - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `pinot` _(required)_ - -### `dsn` - -_[string]_ - DSN(Data Source Name) for the Pinot connection _(required)_ - -### `username` - -_[string]_ - Username for authenticating with Pinot - -### `password` - -_[string]_ - Password for authenticating with Pinot - -### `broker_host` - -_[string]_ - Hostname of the Pinot broker _(required)_ - -### `broker_port` - -_[integer]_ - Port number for the Pinot broker - -### `controller_host` - -_[string]_ - Hostname of the Pinot controller _(required)_ - -### `controller_port` - -_[integer]_ - Port number for the Pinot controller - -### `ssl` - -_[boolean]_ - Enable SSL connection to Pinot - -### `log_queries` - -_[boolean]_ - Log raw SQL queries executed through Pinot - -### `max_open_conns` - -_[integer]_ - Maximum number of open connections to the Pinot database - -## postgres - -Configuration properties specific to the postgres - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `postgres` _(required)_ - -### `dsn` - -_[string]_ - DSN(Data Source Name) for the postgres connection - -### `host` - -_[string]_ - Hostname of the Postgres server - -### `port` - -_[string]_ - Port number for the Postgres server - -### `dbname` - -_[string]_ - Name of the Postgres database - -### `user` - -_[string]_ - Username for authentication - -### `password` - -_[string]_ - Password for authentication - -### `sslmode` - -_[string]_ - SSL mode can be disable, allow, prefer or require - -## redshift - -Configuration properties specific to the redshift - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `redshift` _(required)_ - -### `aws_access_key_id` - -_[string]_ - AWS Access Key ID used for authenticating with Redshift. _(required)_ - -### `aws_secret_access_key` - -_[string]_ - AWS Secret Access Key used for authenticating with Redshift. _(required)_ - -### `aws_access_token` - -_[string]_ - AWS Session Token for temporary credentials (optional). - -### `region` - -_[string]_ - AWS region where the Redshift cluster or workgroup is hosted (e.g., 'us-east-1'). - -### `database` - -_[string]_ - Name of the Redshift database to query. _(required)_ - -### `workgroup` - -_[string]_ - Workgroup name for Redshift Serverless, in case of provisioned Redshift clusters use 'cluster_identifier'. - -### `cluster_identifier` - -_[string]_ - Cluster identifier for provisioned Redshift clusters, in case of Redshift Serverless use 'workgroup' . - -## s3 - -Configuration properties specific to the s3 - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `s3` _(required)_ - -### `aws_access_key_id` - -_[string]_ - AWS Access Key ID used for authentication - -### `aws_secret_access_key` - -_[string]_ - AWS Secret Access Key used for authentication - -### `aws_access_token` - -_[string]_ - Optional AWS session token for temporary credentials - -### `bucket` - -_[string]_ - Name of s3 bucket _(required)_ - -### `endpoint` - -_[string]_ - Optional custom endpoint URL for S3-compatible storage - -### `region` - -_[string]_ - AWS region of the S3 bucket - -### `allow_host_access` - -_[boolean]_ - Allow access to host environment configuration - -### `retain_files` - -_[boolean]_ - Whether to retain intermediate files after processing - -## salesforce - -Configuration properties specific to the salesforce - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `salesforce` _(required)_ - -### `username` - -_[string]_ - Salesforce account username _(required)_ - -### `password` - -_[string]_ - Salesforce account password (secret) - -### `key` - -_[string]_ - Authentication key for Salesforce (secret) - -### `endpoint` - -_[string]_ - Salesforce API endpoint URL _(required)_ - -### `client_id` - -_[string]_ - Client ID used for Salesforce OAuth authentication - -## slack - -Configuration properties specific to the slack - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `slack` _(required)_ - -### `bot_token` - -_[string]_ - Bot token used for authenticating Slack API requests _(required)_ - -## snowflake - -Configuration properties specific to the snowflake - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `snowflake` _(required)_ - -### `dsn` - -_[string]_ - DSN (Data Source Name) for the Snowflake connection _(required)_ - -### `parallel_fetch_limit` - -_[integer]_ - Maximum number of concurrent fetches during query execution - -## sqlite - -Configuration properties specific to the sqlite - -### `driver` - -_[string]_ - Refers to the driver type and must be driver `sqlite` _(required)_ - -### `dsn` - -_[string]_ - DSN(Data Source Name) for the sqlite connection _(required)_ \ No newline at end of file diff --git a/docs/docs/hidden/yaml/connectors.md b/docs/docs/hidden/yaml/connectors.md index 55f65d385d0..783425659c6 100644 --- a/docs/docs/hidden/yaml/connectors.md +++ b/docs/docs/hidden/yaml/connectors.md @@ -1,12 +1,58 @@ --- note: GENERATED. DO NOT EDIT. -title: Connectors YAML -sidebar_position: 34 +title: Connector YAML +sidebar_position: 31 --- Connector YAML files define how Rill connects to external data sources and OLAP engines. Each connector specifies a driver type and its required connection parameters. +## Available Connector Types + +Choose from the following connector types based on your data source: + +### OLAP Engines + +- [**DuckDB**](#duckdb) - Embedded DuckDB engine (default) +- [**ClickHouse**](#clickhouse) - ClickHouse analytical database +- [**MotherDuck**](#motherduck) - MotherDuck cloud database +- [**Druid**](#druid) - Apache Druid +- [**Pinot**](#pinot) - Apache Pinot + +### Data Warehouses + +- [**Snowflake**](#snowflake) - Snowflake data warehouse +- [**BigQuery**](#bigquery) - Google BigQuery +- [**Redshift**](#redshift) - Amazon Redshift +- [**Athena**](#athena) - Amazon Athena + +### Databases + +- [**PostgreSQL**](#postgres) - PostgreSQL databases +- [**MySQL**](#mysql) - MySQL databases +- [**SQLite**](#sqlite) - SQLite databases + +### Cloud Storage + +- [**GCS**](#gcs) - Google Cloud Storage +- [**S3**](#s3) - Amazon S3 storage +- [**Azure**](#azure) - Azure Blob Storage + +### Other + +- [**HTTPS**](#https) - Public files via HTTP/HTTPS +- [**Salesforce**](#salesforce) - Salesforce data +- [**Slack**](#slack) - Slack data +- [**Local File**](#local_file) - Local file system + +:::warning Security Recommendation +For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/build/credentials/) for complete setup instructions. +::: + +## Connector Details + + + ## Properties ### `type` @@ -17,726 +63,864 @@ _[string]_ - Refers to the resource type and must be `connector` _(required)_ _[string]_ - The type of connector, see [available connectors](#available-connector-types) (required) _(required)_ -## One of Properties Options -- [Athena](#athena) -- [Azure](#azure) -- [BigQuery](#bigquery) -- [ClickHouse](#clickhouse) -- [Druid](#druid) -- [DuckDB](#duckdb) -- [GCS](#gcs) -- [HTTPS](#https) -- [MotherDuck](#motherduck) -- [MySQL](#mysql) -- [Pinot](#pinot) -- [Postgres](#postgres) -- [Redshift](#redshift) -- [S3](#s3) -- [Salesforce](#salesforce) -- [Slack](#slack) -- [Snowflake](#snowflake) -- [SQLite](#sqlite) - -#### Option 1: Athena - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the athena - ## Athena Configuration properties specific to the athena -### `driver` +```yaml +type: connector # Must be `connector` (required) +driver: athena # Must be `athena` _(required)_ + +aws_access_key_id: "myawsaccesskey" # AWS Access Key ID used for authentication. Required when using static credentials directly or as base credentials for assuming a role. +aws_secret_access_key: "myawssecretkey"# AWS Secret Access Key paired with the Access Key ID. Required when using static credentials directly or as base credentials for assuming a role. +aws_access_token: "mytemporarytoken" # AWS session token used with temporary credentials. Required only if the Access Key and Secret Key are part of a temporary session credentials. +role_arn: "arn:aws:iam::123456789012:role/MyRole"# ARN of the IAM role to assume. When specified, the SDK uses the base credentials to call STS AssumeRole and obtain temporary credentials scoped to this role. +role_session_name: "MySession" # Session name to associate with the STS AssumeRole session. Used only if 'role_arn' is specified. Useful for identifying and auditing the session. +external_id: "MyExternalID" # External ID required by some roles when assuming them, typically for cross-account access. Used only if 'role_arn' is specified and the role's trust policy requires it. +workgroup: "primary" # Athena workgroup to use for query execution. Defaults to 'primary' if not specified. +output_location: "s3://my-bucket/athena-output/"# S3 URI where Athena query results should be stored (e.g., s3://your-bucket/athena/results/). Optional if the selected workgroup has a default result configuration. +aws_region: "us-east-1" # AWS region where Athena and the result S3 bucket are located (e.g., us-east-1). Defaults to 'us-east-1' if not specified. +allow_host_access: true # Allow the Athena client to access host environment configurations such as environment variables or local AWS credential files. Defaults to true, enabling use of credentials and settings from the host environment unless explicitly disabled. +``` + + + +#### `driver` _[string]_ - Refers to the driver type and must be driver `athena` _(required)_ -### `aws_access_key_id` +#### `aws_access_key_id` _[string]_ - AWS Access Key ID used for authentication. Required when using static credentials directly or as base credentials for assuming a role. -### `aws_secret_access_key` +#### `aws_secret_access_key` _[string]_ - AWS Secret Access Key paired with the Access Key ID. Required when using static credentials directly or as base credentials for assuming a role. -### `aws_access_token` +#### `aws_access_token` _[string]_ - AWS session token used with temporary credentials. Required only if the Access Key and Secret Key are part of a temporary session credentials. -### `role_arn` +#### `role_arn` _[string]_ - ARN of the IAM role to assume. When specified, the SDK uses the base credentials to call STS AssumeRole and obtain temporary credentials scoped to this role. -### `role_session_name` +#### `role_session_name` _[string]_ - Session name to associate with the STS AssumeRole session. Used only if 'role_arn' is specified. Useful for identifying and auditing the session. -### `external_id` +#### `external_id` _[string]_ - External ID required by some roles when assuming them, typically for cross-account access. Used only if 'role_arn' is specified and the role's trust policy requires it. -### `workgroup` +#### `workgroup` _[string]_ - Athena workgroup to use for query execution. Defaults to 'primary' if not specified. -### `output_location` +#### `output_location` _[string]_ - S3 URI where Athena query results should be stored (e.g., s3://your-bucket/athena/results/). Optional if the selected workgroup has a default result configuration. -### `aws_region` +#### `aws_region` _[string]_ - AWS region where Athena and the result S3 bucket are located (e.g., us-east-1). Defaults to 'us-east-1' if not specified. -### `allow_host_access` +#### `allow_host_access` _[boolean]_ - Allow the Athena client to access host environment configurations such as environment variables or local AWS credential files. Defaults to true, enabling use of credentials and settings from the host environment unless explicitly disabled. -#### Option 2: Azure +## Azure -**Type:** _[object]_ +Configuration properties specific to the azure -**Description:** Configuration properties specific to the azure +```yaml +type: connector # Must be `connector` (required) +driver: azure # Must be `azure` _(required)_ -## Azure +azure_storage_account: "myaccount" # Azure storage account name +azure_storage_key: "example_value" # Azure storage access key +azure_storage_sas_token: "mytemporarytoken"# Optional azure SAS token for authentication +azure_storage_connection_string: "example_value"# Optional azure connection string for storage account +azure_storage_bucket: "my-bucket" # Name of the Azure Blob Storage container (equivalent to an S3 bucket) _(required)_ +allow_host_access: true # Allow access to host environment configuration +``` -Configuration properties specific to the azure -### `driver` + +#### `driver` _[string]_ - Refers to the driver type and must be driver `azure` _(required)_ -### `azure_storage_account` +#### `azure_storage_account` _[string]_ - Azure storage account name -### `azure_storage_key` +#### `azure_storage_key` _[string]_ - Azure storage access key -### `azure_storage_sas_token` +#### `azure_storage_sas_token` _[string]_ - Optional azure SAS token for authentication -### `azure_storage_connection_string` +#### `azure_storage_connection_string` _[string]_ - Optional azure connection string for storage account -### `azure_storage_bucket` +#### `azure_storage_bucket` _[string]_ - Name of the Azure Blob Storage container (equivalent to an S3 bucket) _(required)_ -### `allow_host_access` +#### `allow_host_access` _[boolean]_ - Allow access to host environment configuration -#### Option 3: BigQuery +## BigQuery -**Type:** _[object]_ +Configuration properties specific to the bigquery -**Description:** Configuration properties specific to the bigquery +```yaml +type: connector # Must be `connector` (required) +driver: bigquery # Must be `bigquery` _(required)_ -## BigQuery +google_application_credentials: "example_value"# Raw contents of the Google Cloud service account key (in JSON format) used for authentication. +project_id: "myproject" # ID of the Google Cloud project to use for BigQuery operations. This can be omitted only if the project ID is included in the service account key. +allow_host_access: true # Enable the BigQuery client to use credentials from the host environment when no service account JSON is provided. This includes Application Default Credentials from environment variables, local credential files, or the Google Compute Engine metadata server. Defaults to true, allowing seamless authentication in GCP environments. +``` -Configuration properties specific to the bigquery -### `driver` + +#### `driver` _[string]_ - Refers to the driver type and must be driver `bigquery` _(required)_ -### `google_application_credentials` +#### `google_application_credentials` _[string]_ - Raw contents of the Google Cloud service account key (in JSON format) used for authentication. -### `project_id` +#### `project_id` _[string]_ - ID of the Google Cloud project to use for BigQuery operations. This can be omitted only if the project ID is included in the service account key. -### `allow_host_access` +#### `allow_host_access` _[boolean]_ - Enable the BigQuery client to use credentials from the host environment when no service account JSON is provided. This includes Application Default Credentials from environment variables, local credential files, or the Google Compute Engine metadata server. Defaults to true, allowing seamless authentication in GCP environments. -#### Option 4: ClickHouse - -**Type:** _[object]_ - -**Description:** Configuration properties specific to the clickhouse - ## ClickHouse Configuration properties specific to the clickhouse -### `driver` +```yaml +type: connector # Must be `connector` (required) +driver: clickhouse # Must be `clickhouse` _(required)_ + +managed: true # `true` means Rill will provision the connector using the default provisioner. `false` disables automatic provisioning. +mode: "example_value" # `read` - Controls the operation mode for the ClickHouse connection. Defaults to 'read' for safe operation with external databases. Set to 'readwrite' to enable model creation and table mutations. Note: When 'managed: true', this is automatically set to 'readwrite'. +dsn: "postgresql://user:pass@localhost:5432/db"# DSN(Data Source Name) for the ClickHouse connection +username: "myusername" # Username for authentication +password: "mypassword" # Password for authentication +host: "localhost" # Host where the ClickHouse instance is running +port: 123 # Port where the ClickHouse instance is accessible +database: "mydatabase" # Name of the ClickHouse database within the cluster +ssl: true # Indicates whether a secured SSL connection is required +cluster: "mycluster" # Cluster name, required for running distributed queries +log_queries: true # Controls whether to log raw SQL queries +settings_override: "example_value" # override the default settings used in queries. example `readonly = 1, session_timezone = 'UTC'` +embed_port: 123 # Port to run ClickHouse locally (0 for random port) +can_scale_to_zero: true # Indicates if the database can scale to zero +max_open_conns: 123 # Maximum number of open connections to the database +max_idle_conns: 123 # Maximum number of idle connections in the pool +dial_timeout: "example_value" # Timeout for dialing the ClickHouse server +conn_max_lifetime: "example_value" # Maximum time a connection may be reused +read_timeout: "example_value" # Maximum time for a connection to read data +``` + + + +#### `driver` _[string]_ - Refers to the driver type and must be driver `clickhouse` _(required)_ -### `managed` +#### `managed` _[boolean]_ - `true` means Rill will provision the connector using the default provisioner. `false` disables automatic provisioning. -### `mode` +#### `mode` _[string]_ - `read` - Controls the operation mode for the ClickHouse connection. Defaults to 'read' for safe operation with external databases. Set to 'readwrite' to enable model creation and table mutations. Note: When 'managed: true', this is automatically set to 'readwrite'. -### `dsn` +#### `dsn` _[string]_ - DSN(Data Source Name) for the ClickHouse connection -### `username` +#### `username` _[string]_ - Username for authentication -### `password` +#### `password` _[string]_ - Password for authentication -### `host` +#### `host` _[string]_ - Host where the ClickHouse instance is running -### `port` +#### `port` _[integer]_ - Port where the ClickHouse instance is accessible -### `database` +#### `database` _[string]_ - Name of the ClickHouse database within the cluster -### `ssl` +#### `ssl` _[boolean]_ - Indicates whether a secured SSL connection is required -### `cluster` +#### `cluster` _[string]_ - Cluster name, required for running distributed queries -### `log_queries` +#### `log_queries` _[boolean]_ - Controls whether to log raw SQL queries -### `settings_override` +#### `settings_override` _[string]_ - override the default settings used in queries. example `readonly = 1, session_timezone = 'UTC'` -### `embed_port` +#### `embed_port` _[integer]_ - Port to run ClickHouse locally (0 for random port) -### `can_scale_to_zero` +#### `can_scale_to_zero` _[boolean]_ - Indicates if the database can scale to zero -### `max_open_conns` +#### `max_open_conns` _[integer]_ - Maximum number of open connections to the database -### `max_idle_conns` +#### `max_idle_conns` _[integer]_ - Maximum number of idle connections in the pool -### `dial_timeout` +#### `dial_timeout` _[string]_ - Timeout for dialing the ClickHouse server -### `conn_max_lifetime` +#### `conn_max_lifetime` _[string]_ - Maximum time a connection may be reused -### `read_timeout` +#### `read_timeout` _[string]_ - Maximum time for a connection to read data -#### Option 5: Druid +## Druid -**Type:** _[object]_ +Configuration properties specific to the druid -**Description:** Configuration properties specific to the druid +```yaml +type: connector # Must be `connector` (required) +driver: druid # Must be `druid` _(required)_ -## Druid +dsn: "postgresql://user:pass@localhost:5432/db"# Data Source Name (DSN) for connecting to Druid _(required)_ +username: "myusername" # Username for authenticating with Druid +password: "mypassword" # Password for authenticating with Druid +host: "localhost" # Hostname of the Druid coordinator or broker +port: 123 # Port number of the Druid service +ssl: true # Enable SSL for secure connection +log_queries: true # Log raw SQL queries sent to Druid +max_open_conns: 123 # Maximum number of open database connections (0 = default, -1 = unlimited) +skip_version_check: true # Skip checking Druid version compatibility +``` -Configuration properties specific to the druid -### `driver` + +#### `driver` _[string]_ - Refers to the driver type and must be driver `druid` _(required)_ -### `dsn` +#### `dsn` _[string]_ - Data Source Name (DSN) for connecting to Druid _(required)_ -### `username` +#### `username` _[string]_ - Username for authenticating with Druid -### `password` +#### `password` _[string]_ - Password for authenticating with Druid -### `host` +#### `host` _[string]_ - Hostname of the Druid coordinator or broker -### `port` +#### `port` _[integer]_ - Port number of the Druid service -### `ssl` +#### `ssl` _[boolean]_ - Enable SSL for secure connection -### `log_queries` +#### `log_queries` _[boolean]_ - Log raw SQL queries sent to Druid -### `max_open_conns` +#### `max_open_conns` _[integer]_ - Maximum number of open database connections (0 = default, -1 = unlimited) -### `skip_version_check` +#### `skip_version_check` _[boolean]_ - Skip checking Druid version compatibility -#### Option 6: DuckDB +## DuckDB + +Configuration properties specific to the duckdb -**Type:** _[object]_ +```yaml +type: connector # Must be `connector` (required) +driver: duckdb # Must be `duckdb` _(required)_ -**Description:** Configuration properties specific to the duckdb +pool_size: 123 # Number of concurrent connections and queries allowed +allow_host_access: true # Whether access to the local environment and file system is allowed +cpu: 123 # Number of CPU cores available to the database +memory_limit_gb: 123 # Amount of memory in GB available to the database +read_write_ratio: 123.45 # Ratio of resources allocated to the read database; used to divide CPU and memory +init_sql: | + INSTALL 'motherduck'; + LOAD 'motherduck'; + SET motherduck_token= '{{ .env.motherduck_token }}'# is executed during database initialization. +conn_init_sql: | + INSTALL 'motherduck'; + LOAD 'motherduck'; + SET motherduck_token= '{{ .env.motherduck_token }}'# is executed when a new connection is initialized. +secrets: "example_value" # Comma-separated list of other connector names to create temporary secrets for in DuckDB before executing a model. +log_queries: true # Whether to log raw SQL queries executed through OLAP +``` -## DuckDB -Configuration properties specific to the duckdb -### `driver` +#### `driver` _[string]_ - Refers to the driver type and must be driver `duckdb` _(required)_ -### `pool_size` +#### `pool_size` _[integer]_ - Number of concurrent connections and queries allowed -### `allow_host_access` +#### `allow_host_access` _[boolean]_ - Whether access to the local environment and file system is allowed -### `cpu` +#### `cpu` _[integer]_ - Number of CPU cores available to the database -### `memory_limit_gb` +#### `memory_limit_gb` _[integer]_ - Amount of memory in GB available to the database -### `read_write_ratio` +#### `read_write_ratio` _[number]_ - Ratio of resources allocated to the read database; used to divide CPU and memory -### `init_sql` +#### `init_sql` _[string]_ - is executed during database initialization. -### `conn_init_sql` +#### `conn_init_sql` _[string]_ - is executed when a new connection is initialized. -### `secrets` +#### `secrets` _[string]_ - Comma-separated list of other connector names to create temporary secrets for in DuckDB before executing a model. -### `log_queries` +#### `log_queries` _[boolean]_ - Whether to log raw SQL queries executed through OLAP -#### Option 7: GCS +## GCS -**Type:** _[object]_ +Configuration properties specific to the gcs -**Description:** Configuration properties specific to the gcs +```yaml +type: connector # Must be `connector` (required) +driver: gcs # Must be `gcs` _(required)_ -## GCS +google_application_credentials: "example_value"# Google Cloud credentials JSON string +bucket: "my-bucket" # Name of gcs bucket _(required)_ +allow_host_access: true # Allow access to host environment configuration +key_id: "example_value" # Optional S3-compatible Key ID when used in compatibility mode +secret: "example_value" # Optional S3-compatible Secret when used in compatibility mode +``` -Configuration properties specific to the gcs -### `driver` + +#### `driver` _[string]_ - Refers to the driver type and must be driver `gcs` _(required)_ -### `google_application_credentials` +#### `google_application_credentials` _[string]_ - Google Cloud credentials JSON string -### `bucket` +#### `bucket` _[string]_ - Name of gcs bucket _(required)_ -### `allow_host_access` +#### `allow_host_access` _[boolean]_ - Allow access to host environment configuration -### `key_id` +#### `key_id` _[string]_ - Optional S3-compatible Key ID when used in compatibility mode -### `secret` +#### `secret` _[string]_ - Optional S3-compatible Secret when used in compatibility mode -#### Option 8: HTTPS +## HTTPS -**Type:** _[object]_ +Configuration properties specific to the https -**Description:** Configuration properties specific to the https +```yaml +type: connector # Must be `connector` (required) +driver: https # Must be `https` _(required)_ -## HTTPS +path: "md:my_db" # The full HTTPS URI to fetch data from _(required)_ +headers: "example_value" # HTTP headers to include in the request +``` -Configuration properties specific to the https -### `driver` + +#### `driver` _[string]_ - Refers to the driver type and must be driver `https` _(required)_ -### `path` +#### `path` _[string]_ - The full HTTPS URI to fetch data from _(required)_ -### `headers` +#### `headers` _[object]_ - HTTP headers to include in the request -#### Option 9: MotherDuck +## MotherDuck -**Type:** _[object]_ +Configuration properties specific to the motherduck -**Description:** Configuration properties specific to the motherduck +```yaml +type: connector # Must be `connector` (required) +driver: duckdb # Must be `duckdb` _(required)_ -## MotherDuck +path: "md:my_db" # Path to your MD database _(required)_ +init_sql: | + INSTALL 'motherduck'; + LOAD 'motherduck'; + SET motherduck_token= '{{ .env.motherduck_token }}'# SQL executed during database initialization. _(required)_ +``` -Configuration properties specific to the motherduck -### `driver` + +#### `driver` _[string]_ - Refers to the driver type and must be driver `duckdb` _(required)_ -### `path` +#### `path` _[string]_ - Path to your MD database _(required)_ -### `init_sql` +#### `init_sql` _[string]_ - SQL executed during database initialization. _(required)_ -#### Option 10: MySQL +## MySQL + +Configuration properties specific to the mysql -**Type:** _[object]_ +```yaml +type: connector # Must be `connector` (required) +driver: mysql # Must be `mysql` _(required)_ -**Description:** Configuration properties specific to the mysql +dsn: "postgresql://user:pass@localhost:5432/db"# DSN(Data Source Name) for the mysql connection +host: "localhost" # Hostname of the MySQL server +port: 123 # Port number for the MySQL server +database: "mydatabase" # Name of the MySQL database +user: "example_value" # Username for authentication +password: "mypassword" # Password for authentication +ssl_mode: "example_value" # SSL mode can be DISABLED, PREFERRED or REQUIRED +``` -## MySQL -Configuration properties specific to the mysql -### `driver` +#### `driver` _[string]_ - Refers to the driver type and must be driver `mysql` _(required)_ -### `dsn` +#### `dsn` _[string]_ - DSN(Data Source Name) for the mysql connection -### `host` +#### `host` _[string]_ - Hostname of the MySQL server -### `port` +#### `port` _[integer]_ - Port number for the MySQL server -### `database` +#### `database` _[string]_ - Name of the MySQL database -### `user` +#### `user` _[string]_ - Username for authentication -### `password` +#### `password` _[string]_ - Password for authentication -### `ssl_mode` +#### `ssl_mode` _[string]_ - SSL mode can be DISABLED, PREFERRED or REQUIRED -#### Option 11: Pinot +## Pinot -**Type:** _[object]_ +Configuration properties specific to the pinot -**Description:** Configuration properties specific to the pinot +```yaml +type: connector # Must be `connector` (required) +driver: pinot # Must be `pinot` _(required)_ -## Pinot +dsn: "postgresql://user:pass@localhost:5432/db"# DSN(Data Source Name) for the Pinot connection _(required)_ +username: "myusername" # Username for authenticating with Pinot +password: "mypassword" # Password for authenticating with Pinot +broker_host: "localhost" # Hostname of the Pinot broker _(required)_ +broker_port: 123 # Port number for the Pinot broker +controller_host: "localhost" # Hostname of the Pinot controller _(required)_ +controller_port: 123 # Port number for the Pinot controller +ssl: true # Enable SSL connection to Pinot +log_queries: true # Log raw SQL queries executed through Pinot +max_open_conns: 123 # Maximum number of open connections to the Pinot database +``` -Configuration properties specific to the pinot -### `driver` + +#### `driver` _[string]_ - Refers to the driver type and must be driver `pinot` _(required)_ -### `dsn` +#### `dsn` _[string]_ - DSN(Data Source Name) for the Pinot connection _(required)_ -### `username` +#### `username` _[string]_ - Username for authenticating with Pinot -### `password` +#### `password` _[string]_ - Password for authenticating with Pinot -### `broker_host` +#### `broker_host` _[string]_ - Hostname of the Pinot broker _(required)_ -### `broker_port` +#### `broker_port` _[integer]_ - Port number for the Pinot broker -### `controller_host` +#### `controller_host` _[string]_ - Hostname of the Pinot controller _(required)_ -### `controller_port` +#### `controller_port` _[integer]_ - Port number for the Pinot controller -### `ssl` +#### `ssl` _[boolean]_ - Enable SSL connection to Pinot -### `log_queries` +#### `log_queries` _[boolean]_ - Log raw SQL queries executed through Pinot -### `max_open_conns` +#### `max_open_conns` _[integer]_ - Maximum number of open connections to the Pinot database -#### Option 12: Postgres +## Postgres -**Type:** _[object]_ +Configuration properties specific to the postgres -**Description:** Configuration properties specific to the postgres +```yaml +type: connector # Must be `connector` (required) +driver: postgres # Must be `postgres` _(required)_ -## Postgres +dsn: "postgresql://user:pass@localhost:5432/db"# DSN(Data Source Name) for the postgres connection +host: "localhost" # Hostname of the Postgres server +port: 5432 # Port number for the Postgres server +dbname: "example_value" # Name of the Postgres database +user: "example_value" # Username for authentication +password: "mypassword" # Password for authentication +sslmode: "example_value" # SSL mode can be disable, allow, prefer or require +``` -Configuration properties specific to the postgres -### `driver` + +#### `driver` _[string]_ - Refers to the driver type and must be driver `postgres` _(required)_ -### `dsn` +#### `dsn` _[string]_ - DSN(Data Source Name) for the postgres connection -### `host` +#### `host` _[string]_ - Hostname of the Postgres server -### `port` +#### `port` _[string]_ - Port number for the Postgres server -### `dbname` +#### `dbname` _[string]_ - Name of the Postgres database -### `user` +#### `user` _[string]_ - Username for authentication -### `password` +#### `password` _[string]_ - Password for authentication -### `sslmode` +#### `sslmode` _[string]_ - SSL mode can be disable, allow, prefer or require -#### Option 13: Redshift +## Redshift -**Type:** _[object]_ +Configuration properties specific to the redshift -**Description:** Configuration properties specific to the redshift +```yaml +type: connector # Must be `connector` (required) +driver: redshift # Must be `redshift` _(required)_ -## Redshift +aws_access_key_id: "myawsaccesskey" # AWS Access Key ID used for authenticating with Redshift. _(required)_ +aws_secret_access_key: "myawssecretkey"# AWS Secret Access Key used for authenticating with Redshift. _(required)_ +aws_access_token: "mytemporarytoken" # AWS Session Token for temporary credentials (optional). +region: "us-east-1" # AWS region where the Redshift cluster or workgroup is hosted (e.g., 'us-east-1'). +database: "mydatabase" # Name of the Redshift database to query. _(required)_ +workgroup: "primary" # Workgroup name for Redshift Serverless, in case of provisioned Redshift clusters use 'cluster_identifier'. +cluster_identifier: "mycluster" # Cluster identifier for provisioned Redshift clusters, in case of Redshift Serverless use 'workgroup' . +``` -Configuration properties specific to the redshift -### `driver` + +#### `driver` _[string]_ - Refers to the driver type and must be driver `redshift` _(required)_ -### `aws_access_key_id` +#### `aws_access_key_id` _[string]_ - AWS Access Key ID used for authenticating with Redshift. _(required)_ -### `aws_secret_access_key` +#### `aws_secret_access_key` _[string]_ - AWS Secret Access Key used for authenticating with Redshift. _(required)_ -### `aws_access_token` +#### `aws_access_token` _[string]_ - AWS Session Token for temporary credentials (optional). -### `region` +#### `region` _[string]_ - AWS region where the Redshift cluster or workgroup is hosted (e.g., 'us-east-1'). -### `database` +#### `database` _[string]_ - Name of the Redshift database to query. _(required)_ -### `workgroup` +#### `workgroup` _[string]_ - Workgroup name for Redshift Serverless, in case of provisioned Redshift clusters use 'cluster_identifier'. -### `cluster_identifier` +#### `cluster_identifier` _[string]_ - Cluster identifier for provisioned Redshift clusters, in case of Redshift Serverless use 'workgroup' . -#### Option 14: S3 +## S3 -**Type:** _[object]_ +Configuration properties specific to the s3 -**Description:** Configuration properties specific to the s3 +```yaml +type: connector # Must be `connector` (required) +driver: s3 # Must be `s3` _(required)_ -## S3 +aws_access_key_id: "myawsaccesskey" # AWS Access Key ID used for authentication +aws_secret_access_key: "myawssecretkey"# AWS Secret Access Key used for authentication +aws_access_token: "mytemporarytoken" # Optional AWS session token for temporary credentials +bucket: "my-bucket" # Name of s3 bucket _(required)_ +endpoint: "https://api.example.com" # Optional custom endpoint URL for S3-compatible storage +region: "us-east-1" # AWS region of the S3 bucket +allow_host_access: true # Allow access to host environment configuration +retain_files: true # Whether to retain intermediate files after processing +``` -Configuration properties specific to the s3 -### `driver` + +#### `driver` _[string]_ - Refers to the driver type and must be driver `s3` _(required)_ -### `aws_access_key_id` +#### `aws_access_key_id` _[string]_ - AWS Access Key ID used for authentication -### `aws_secret_access_key` +#### `aws_secret_access_key` _[string]_ - AWS Secret Access Key used for authentication -### `aws_access_token` +#### `aws_access_token` _[string]_ - Optional AWS session token for temporary credentials -### `bucket` +#### `bucket` _[string]_ - Name of s3 bucket _(required)_ -### `endpoint` +#### `endpoint` _[string]_ - Optional custom endpoint URL for S3-compatible storage -### `region` +#### `region` _[string]_ - AWS region of the S3 bucket -### `allow_host_access` +#### `allow_host_access` _[boolean]_ - Allow access to host environment configuration -### `retain_files` +#### `retain_files` _[boolean]_ - Whether to retain intermediate files after processing -#### Option 15: Salesforce +## Salesforce -**Type:** _[object]_ +Configuration properties specific to the salesforce -**Description:** Configuration properties specific to the salesforce +```yaml +type: connector # Must be `connector` (required) +driver: salesforce # Must be `salesforce` _(required)_ -## Salesforce +username: "myusername" # Salesforce account username _(required)_ +password: "mypassword" # Salesforce account password (secret) +key: "example_value" # Authentication key for Salesforce (secret) +endpoint: "https://api.example.com" # Salesforce API endpoint URL _(required)_ +client_id: "example_value" # Client ID used for Salesforce OAuth authentication +``` -Configuration properties specific to the salesforce -### `driver` + +#### `driver` _[string]_ - Refers to the driver type and must be driver `salesforce` _(required)_ -### `username` +#### `username` _[string]_ - Salesforce account username _(required)_ -### `password` +#### `password` _[string]_ - Salesforce account password (secret) -### `key` +#### `key` _[string]_ - Authentication key for Salesforce (secret) -### `endpoint` +#### `endpoint` _[string]_ - Salesforce API endpoint URL _(required)_ -### `client_id` +#### `client_id` _[string]_ - Client ID used for Salesforce OAuth authentication -#### Option 16: Slack +## Slack -**Type:** _[object]_ +Configuration properties specific to the slack -**Description:** Configuration properties specific to the slack +```yaml +type: connector # Must be `connector` (required) +driver: slack # Must be `slack` _(required)_ -## Slack +bot_token: "mytemporarytoken" # Bot token used for authenticating Slack API requests _(required)_ +``` -Configuration properties specific to the slack -### `driver` + +#### `driver` _[string]_ - Refers to the driver type and must be driver `slack` _(required)_ -### `bot_token` +#### `bot_token` _[string]_ - Bot token used for authenticating Slack API requests _(required)_ -#### Option 17: Snowflake +## Snowflake -**Type:** _[object]_ +Configuration properties specific to the snowflake -**Description:** Configuration properties specific to the snowflake +```yaml +type: connector # Must be `connector` (required) +driver: snowflake # Must be `snowflake` _(required)_ -## Snowflake +dsn: "postgresql://user:pass@localhost:5432/db"# DSN (Data Source Name) for the Snowflake connection _(required)_ +parallel_fetch_limit: 123 # Maximum number of concurrent fetches during query execution +``` -Configuration properties specific to the snowflake -### `driver` + +#### `driver` _[string]_ - Refers to the driver type and must be driver `snowflake` _(required)_ -### `dsn` +#### `dsn` _[string]_ - DSN (Data Source Name) for the Snowflake connection _(required)_ -### `parallel_fetch_limit` +#### `parallel_fetch_limit` _[integer]_ - Maximum number of concurrent fetches during query execution -#### Option 18: SQLite +## SQLite -**Type:** _[object]_ +Configuration properties specific to the sqlite -**Description:** Configuration properties specific to the sqlite +```yaml +type: connector # Must be `connector` (required) +driver: sqlite # Must be `sqlite` _(required)_ -## SQLite +dsn: "postgresql://user:pass@localhost:5432/db"# DSN(Data Source Name) for the sqlite connection _(required)_ +``` -Configuration properties specific to the sqlite -### `driver` + +#### `driver` _[string]_ - Refers to the driver type and must be driver `sqlite` _(required)_ -### `dsn` +#### `dsn` _[string]_ - DSN(Data Source Name) for the sqlite connection _(required)_ \ No newline at end of file diff --git a/docs/docs/hidden/yaml/explore-dashboards.md b/docs/docs/hidden/yaml/explore-dashboards.md index 7361ec9ad44..428026f6b55 100644 --- a/docs/docs/hidden/yaml/explore-dashboards.md +++ b/docs/docs/hidden/yaml/explore-dashboards.md @@ -1,7 +1,7 @@ --- note: GENERATED. DO NOT EDIT. -title: Explore Dashboards YAML -sidebar_position: 35 +title: Explore Dashboard YAML +sidebar_position: 37 --- In your Rill project directory, create a explore dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. diff --git a/docs/docs/hidden/yaml/explore.md b/docs/docs/hidden/yaml/explore.md deleted file mode 100644 index 727b6b5c202..00000000000 --- a/docs/docs/hidden/yaml/explore.md +++ /dev/null @@ -1,215 +0,0 @@ ---- -note: GENERATED. DO NOT EDIT. -title: Explore YAML -sidebar_position: 36 ---- - -In your Rill project directory, create a explore dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. - -## Properties - -### `type` - -_[string]_ - Refers to the resource type and must be `explore` _(required)_ - -### `display_name` - -_[string]_ - Refers to the display name for the explore dashboard - -### `description` - -_[string]_ - Refers to the description of the explore dashboard - -### `banner` - -_[string]_ - Refers to the custom banner displayed at the header of an explore dashboard - -### `metrics_view` - -_[string]_ - Refers to the metrics view resource - -### `dimensions` - -_[oneOf]_ - List of dimension names. Use '*' to select all dimensions (default) - - - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection - - - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection - - - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion - - - **`regex`** - _[string]_ - Select dimensions using a regular expression - - - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - - - **`exclude`** - _[object]_ - Select all dimensions except those listed here - -### `measures` - -_[oneOf]_ - List of measure names. Use ''*'' to select all measures (default) - - - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection - - - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection - - - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion - - - **`regex`** - _[string]_ - Select dimensions using a regular expression - - - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - - - **`exclude`** - _[object]_ - Select all dimensions except those listed here - -### `theme` - -_[oneOf]_ - Name of the theme to use. Only one of theme and embedded_theme can be set. - - - **option 1** - _[string]_ - Name of an existing theme to apply to the dashboard - - - **option 2** - _[object]_ - Inline theme configuration. - - - **`colors`** - _[object]_ - Used to override the dashboard colors. Either primary or secondary color must be provided. - - - **`primary`** - _[string]_ - Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). - - - **`secondary`** - _[string]_ - Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. - -### `time_ranges` - -_[array of oneOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' - - - **option 1** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection - - - **option 2** - _[object]_ - Object containing time range and comparison configuration - - - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ - - - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) - - - **option 1** - _[string]_ - Offset string only (range is inferred) - - - **option 2** - _[object]_ - Object containing offset and range configuration for time comparison - - - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) - - - **`range`** - _[string]_ - Custom time range for comparison period - -### `time_zones` - -_[array of string]_ - Refers to the time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) - -### `lock_time_zone` - -_[boolean]_ - When true, the dashboard will be locked to the first time provided in the time_zones list. When no time_zones are provided, the dashboard will be locked to UTC - -### `allow_custom_time_range` - -_[boolean]_ - Defaults to true, when set to false it will hide the ability to set a custom time range for the user. - -### `defaults` - -_[object]_ - defines the defaults YAML struct - - - **`dimensions`** - _[oneOf]_ - Provides the default dimensions to load on viewing the dashboard - - - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection - - - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection - - - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion - - - **`regex`** - _[string]_ - Select dimensions using a regular expression - - - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - - - **`exclude`** - _[object]_ - Select all dimensions except those listed here - - - **`measures`** - _[oneOf]_ - Provides the default measures to load on viewing the dashboard - - - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection - - - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection - - - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion - - - **`regex`** - _[string]_ - Select dimensions using a regular expression - - - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - - - **`exclude`** - _[object]_ - Select all dimensions except those listed here - - - **`time_range`** - _[string]_ - Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) - - - **`comparison_mode`** - _[string]_ - Controls how to compare current data with historical or categorical baselines. Options: `none` (no comparison), `time` (compares with past based on default_time_range), `dimension` (compares based on comparison_dimension values) - - - **`comparison_dimension`** - _[string]_ - for dimension mode, specify the comparison dimension by name - -### `embeds` - -_[object]_ - Configuration options for embedded dashboard views - - - **`hide_pivot`** - _[boolean]_ - When true, hides the pivot table view in embedded mode - -### `security` - -_[object]_ - Defines security rules and access control policies for resources - - - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. - - - **option 1** - _[string]_ - SQL expression that evaluates to a boolean to determine access - - - **option 2** - _[boolean]_ - Direct boolean value to allow or deny access - - - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause - - - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded - - - **`if`** - _[string]_ - Expression to decide if the column should be included or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ - - - **`names`** - _[anyOf]_ - List of fields to include. Should match the name of one of the dashboard's dimensions or measures _(required)_ - - - **option 1** - _[array of string]_ - List of specific field names to include - - - **option 2** - _[string]_ - Wildcard '*' to include all fields - - - **`exclude`** - _[array of object]_ - List of dimension or measure names to exclude from the dashboard. If exclude is defined all other dimensions and measures are included - - - **`if`** - _[string]_ - Expression to decide if the column should be excluded or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ - - - **`names`** - _[anyOf]_ - List of fields to exclude. Should match the name of one of the dashboard's dimensions or measures _(required)_ - - - **option 1** - _[array of string]_ - List of specific field names to exclude - - - **option 2** - _[string]_ - Wildcard '*' to exclude all fields - - - **`rules`** - _[array of object]_ - List of detailed security rules that can be used to define complex access control policies - - - **`type`** - _[string]_ - Type of security rule - access (overall access), field_access (field-level access), or row_filter (row-level filtering) _(required)_ - - - **`action`** - _[string]_ - Whether to allow or deny access for this rule - - - **`if`** - _[string]_ - Conditional expression that determines when this rule applies. Must be a valid SQL expression that evaluates to a boolean - - - **`names`** - _[array of string]_ - List of field names this rule applies to (for field_access type rules) - - - **`all`** - _[boolean]_ - When true, applies the rule to all fields (for field_access type rules) - - - **`sql`** - _[string]_ - SQL expression for row filtering (for row_filter type rules) - -## Common Properties - -### `name` - -_[string]_ - Name is usually inferred from the filename, but can be specified manually. - -### `refs` - -_[array of string]_ - List of resource references - -### `dev` - -_[object]_ - Overrides any properties in development environment. - -### `prod` - -_[object]_ - Overrides any properties in production environment. \ No newline at end of file diff --git a/docs/docs/hidden/yaml/index.md b/docs/docs/hidden/yaml/index.md index bbe5fa94aec..701f727ea3f 100644 --- a/docs/docs/hidden/yaml/index.md +++ b/docs/docs/hidden/yaml/index.md @@ -28,12 +28,14 @@ For more information about using Git or cloning projects locally, please see our ## Project files types -- [Alerts YAML](alerts.md) -- [APIs YAML](apis.md) -- [Canvas Dashboards YAML](canvas-dashboards.md) -- [Connectors YAML](connectors.md) -- [Explore Dashboards YAML](explore-dashboards.md) -- [Metrics Views YAML](metrics-views.md) -- [Advanced Models YAML](advanced-models.md) -- [Themes YAML](themes.md) -- [Project YAML](project.md) \ No newline at end of file +- [Connector YAML](connectors.md) +- [Source YAML](sources.md) +- [Model SQL](models.md) +- [Models YAML](advanced-models.md) +- [Metrics View YAML](metrics-views.md) +- [Canvas Dashboard YAML](canvas-dashboards.md) +- [Explore Dashboard YAML](explore-dashboards.md) +- [Alert YAML](alerts.md) +- [API YAML](apis.md) +- [Theme YAML](themes.md) +- [Project YAML](rillyaml.md) \ No newline at end of file diff --git a/docs/docs/hidden/yaml/metrics-views.md b/docs/docs/hidden/yaml/metrics-views.md index 61730e64d50..89b13eb08eb 100644 --- a/docs/docs/hidden/yaml/metrics-views.md +++ b/docs/docs/hidden/yaml/metrics-views.md @@ -1,7 +1,7 @@ --- note: GENERATED. DO NOT EDIT. -title: Metrics Views YAML -sidebar_position: 36 +title: Metrics View YAML +sidebar_position: 35 --- In your Rill project directory, create a metrics view, `.yaml`, file in the `metrics` directory. Rill will ingest the metric view definition next time you run `rill start`. diff --git a/docs/docs/hidden/yaml/metrics-view.md b/docs/docs/hidden/yaml/metrics_views.md similarity index 74% rename from docs/docs/hidden/yaml/metrics-view.md rename to docs/docs/hidden/yaml/metrics_views.md index f712e1939ef..dcd196ee12f 100644 --- a/docs/docs/hidden/yaml/metrics-view.md +++ b/docs/docs/hidden/yaml/metrics_views.md @@ -1,7 +1,7 @@ --- note: GENERATED. DO NOT EDIT. -title: Metrics View YAML -sidebar_position: 37 +title: Metrics Views YAML +sidebar_position: 35 --- In your Rill project directory, create a metrics view, `.yaml`, file in the `metrics` directory. Rill will ingest the metric view definition next time you run `rill start`. @@ -10,7 +10,7 @@ In your Rill project directory, create a metrics view, `.yaml`, fi ### `type` -_[string]_ - Refers to the resource type and must be `metrics_view` _(required)_ +_[string]_ - Refers to the resource type and must be `metrics_view` ### `display_name` @@ -163,51 +163,9 @@ _[array of object]_ - Used to define the numeric aggregates of columns from your - **`treat_nulls_as`** - _[string]_ - used to configure what value to fill in for missing time buckets. This also works generally as COALESCING over non empty time buckets. -### `security` +### `required` -_[object]_ - Defines security rules and access control policies for resources - - - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. - - - **option 1** - _[string]_ - SQL expression that evaluates to a boolean to determine access - - - **option 2** - _[boolean]_ - Direct boolean value to allow or deny access - - - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause - - - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded - - - **`if`** - _[string]_ - Expression to decide if the column should be included or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ - - - **`names`** - _[anyOf]_ - List of fields to include. Should match the name of one of the dashboard's dimensions or measures _(required)_ - - - **option 1** - _[array of string]_ - List of specific field names to include - - - **option 2** - _[string]_ - Wildcard '*' to include all fields - - - **`exclude`** - _[array of object]_ - List of dimension or measure names to exclude from the dashboard. If exclude is defined all other dimensions and measures are included - - - **`if`** - _[string]_ - Expression to decide if the column should be excluded or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ - - - **`names`** - _[anyOf]_ - List of fields to exclude. Should match the name of one of the dashboard's dimensions or measures _(required)_ - - - **option 1** - _[array of string]_ - List of specific field names to exclude - - - **option 2** - _[string]_ - Wildcard '*' to exclude all fields - - - **`rules`** - _[array of object]_ - List of detailed security rules that can be used to define complex access control policies - - - **`type`** - _[string]_ - Type of security rule - access (overall access), field_access (field-level access), or row_filter (row-level filtering) _(required)_ - - - **`action`** - _[string]_ - Whether to allow or deny access for this rule - - - **`if`** - _[string]_ - Conditional expression that determines when this rule applies. Must be a valid SQL expression that evaluates to a boolean - - - **`names`** - _[array of string]_ - List of field names this rule applies to (for field_access type rules) - - - **`all`** - _[boolean]_ - When true, applies the rule to all fields (for field_access type rules) - - - **`sql`** - _[string]_ - SQL expression for row filtering (for row_filter type rules) +_[no type]_ - (no description) ## Common Properties diff --git a/docs/docs/hidden/yaml/model.md b/docs/docs/hidden/yaml/model.md deleted file mode 100644 index d0cf32a661c..00000000000 --- a/docs/docs/hidden/yaml/model.md +++ /dev/null @@ -1,398 +0,0 @@ ---- -note: GENERATED. DO NOT EDIT. -title: Model YAML -sidebar_position: 38 ---- - -## Properties - -### `type` - -_[string]_ - Refers to the resource type and must be `model` _(required)_ - -### `refresh` - -_[object]_ - Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying model data - - - **`cron`** - _[string]_ - A cron expression that defines the execution schedule - - - **`time_zone`** - _[string]_ - Time zone to interpret the schedule in (e.g., 'UTC', 'America/Los_Angeles'). - - - **`disable`** - _[boolean]_ - If true, disables the resource without deleting it. - - - **`ref_update`** - _[boolean]_ - If true, allows the resource to run when a dependency updates. - - - **`run_in_dev`** - _[boolean]_ - If true, allows the schedule to run in development mode. - -### `connector` - -_[string]_ - Refers to the connector type or [named connector](./connector.md#name) for the source. - - -### `sql` - -_[string]_ - Raw SQL query to run against source _(required)_ - -### `timeout` - -_[string]_ - The maximum time to wait for model ingestion - -### `incremental` - -_[boolean]_ - whether incremental modeling is required (optional) - -### `change_mode` - -_[string]_ - Configure how changes to the model specifications are applied (optional). 'reset' will drop and recreate the model automatically, 'manual' will require a manual full or incremental refresh to apply changes, and 'patch' will switch to the new logic without re-processing historical data (only applies for incremental models). - -### `state` - -_[oneOf]_ - Refers to the explicitly defined state of your model, cannot be used with partitions (optional) - - - **option 1** - _[object]_ - Executes a raw SQL query against the project's data models. - - - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - - - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. - - - **option 2** - _[object]_ - Executes a SQL query that targets a defined metrics view. - - - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - - - **option 3** - _[object]_ - Calls a custom API defined in the project to compute data. - - - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - - - **`args`** - _[object]_ - Arguments to pass to the custom API. - - - **option 4** - _[object]_ - Uses a file-matching pattern (glob) to query data from a connector. - - - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - - - **option 1** - _[string]_ - A simple file path/glob pattern as a string. - - - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. - - - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. - - - **option 5** - _[object]_ - Uses the status of a resource as data. - - - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - - - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. - -### `partitions` - -_[oneOf]_ - Refers to the how your data is partitioned, cannot be used with state. (optional) - - - **option 1** - _[object]_ - Executes a raw SQL query against the project's data models. - - - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - - - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. - - - **option 2** - _[object]_ - Executes a SQL query that targets a defined metrics view. - - - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - - - **option 3** - _[object]_ - Calls a custom API defined in the project to compute data. - - - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - - - **`args`** - _[object]_ - Arguments to pass to the custom API. - - - **option 4** - _[object]_ - Uses a file-matching pattern (glob) to query data from a connector. - - - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - - - **option 1** - _[string]_ - A simple file path/glob pattern as a string. - - - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. - - - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. - - - **option 5** - _[object]_ - Uses the status of a resource as data. - - - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - - - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. - -### `materialize` - -_[boolean]_ - models will be materialized in olap - -### `partitions_watermark` - -_[string]_ - Refers to a customizable timestamp that can be set to check if an object has been updated (optional). - -### `partitions_concurrency` - -_[integer]_ - Refers to the number of concurrent partitions that can be read at the same time (optional). - -### `stage` - -_[object]_ - in the case of staging models, where an input source does not support direct write to the output and a staging table is required - - - **`connector`** - _[string]_ - Refers to the connector type for the staging table _(required)_ - -### `output` - -_[object]_ - to define the properties of output - - - **`table`** - _[string]_ - Name of the output table. If not specified, the model name is used. - - - **`materialize`** - _[boolean]_ - Whether to materialize the model as a table or view - - - **`connector`** - _[string]_ - Refers to the connector type for the output table. Can be `clickhouse` or `duckdb` and their named connector - - - **`incremental_strategy`** - _[string]_ - Strategy to use for incremental updates. Can be 'append', 'merge' or 'partition_overwrite' - - - **`unique_key`** - _[array of string]_ - List of columns that uniquely identify a row for merge strategy - - - **`partition_by`** - _[string]_ - Column or expression to partition the table by - - **Additional properties for `output` when `connector` is `clickhouse`** - - - **`type`** - _[string]_ - Type to materialize the model into. Can be 'TABLE', 'VIEW' or 'DICTIONARY' - - - **`columns`** - _[string]_ - Column names and types. Can also include indexes. If unspecified, detected from the query. - - - **`engine_full`** - _[string]_ - Full engine definition in SQL format. Can include partition keys, order, TTL, etc. - - - **`engine`** - _[string]_ - Table engine to use. Default is MergeTree - - - **`order_by`** - _[string]_ - ORDER BY clause. - - - **`partition_by`** - _[string]_ - Partition BY clause. - - - **`primary_key`** - _[string]_ - PRIMARY KEY clause. - - - **`sample_by`** - _[string]_ - SAMPLE BY clause. - - - **`ttl`** - _[string]_ - TTL settings for the table or columns. - - - **`table_settings`** - _[string]_ - Table-specific settings. - - - **`query_settings`** - _[string]_ - Settings used in insert/create table as select queries. - - - **`distributed_settings`** - _[string]_ - Settings for distributed table. - - - **`distributed_sharding_key`** - _[string]_ - Sharding key for distributed table. - - - **`dictionary_source_user`** - _[string]_ - User for accessing the source dictionary table (used if type is DICTIONARY). - - - **`dictionary_source_password`** - _[string]_ - Password for the dictionary source user. - -## Common Properties - -### `name` - -_[string]_ - Name is usually inferred from the filename, but can be specified manually. - -### `refs` - -_[array of string]_ - List of resource references - -### `dev` - -_[object]_ - Overrides any properties in development environment. - -### `prod` - -_[object]_ - Overrides any properties in production environment. - -## Additional properties when `connector` is `athena` or [named connector](./connector.md#name) for athena - -### `output_location` - -_[string]_ - Output location for query results in S3. - -### `workgroup` - -_[string]_ - AWS Athena workgroup to use for queries. - -### `region` - -_[string]_ - AWS region to connect to Athena and the output location. - -## Additional properties when `connector` is `azure` or [named connector](./connector.md#name) of azure - -### `path` - -_[string]_ - Path to the source - -### `account` - -_[string]_ - Account identifier - -### `uri` - -_[string]_ - Source URI - -### `extract` - -_[object]_ - Arbitrary key-value pairs for extraction settings - -### `glob` - -_[object]_ - Settings related to glob file matching. - - - **`max_total_size`** - _[integer]_ - Maximum total size (in bytes) matched by glob - - - **`max_objects_matched`** - _[integer]_ - Maximum number of objects matched by glob - - - **`max_objects_listed`** - _[integer]_ - Maximum number of objects listed in glob - - - **`page_size`** - _[integer]_ - Page size for glob listing - -### `batch_size` - -_[string]_ - Size of a batch (e.g., '100MB') - -## Additional properties when `connector` is `bigquery` or [named connector](./connector.md#name) of bigquery - -### `project_id` - -_[string]_ - ID of the BigQuery project. - -## Additional properties when `connector` is `duckdb` or [named connector](./connector.md#name) of duckdb - -### `path` - -_[string]_ - Path to the data source. - -### `format` - -_[string]_ - Format of the data source (e.g., csv, json, parquet). - -### `pre_exec` - -_[string]_ - refers to SQL queries to run before the main query, available for DuckDB-based models. _(optional)_. Ensure `pre_exec` queries are idempotent. Use `IF NOT EXISTS` statements when applicable. - -### `post_exec` - -_[string]_ - refers to a SQL query that is run after the main query, available for DuckDB-based models. _(optional)_. Ensure `post_exec` queries are idempotent. Use `IF EXISTS` statements when applicable. - -```yaml -pre_exec: ATTACH IF NOT EXISTS 'dbname=postgres host=localhost port=5432 user=postgres password=postgres' AS postgres_db (TYPE POSTGRES); -sql: SELECT * FROM postgres_query('postgres_db', 'SELECT * FROM USERS') -post_exec: DETACH DATABASE IF EXISTS postgres_db -``` - -## Additional properties when `connector` is `gcs` or [named connector](./connector.md#name) of gcs - -### `path` - -_[string]_ - Path to the source - -### `uri` - -_[string]_ - Source URI - -### `extract` - -_[object]_ - key-value pairs for extraction settings - -### `glob` - -_[object]_ - Settings related to glob file matching. - - - **`max_total_size`** - _[integer]_ - Maximum total size (in bytes) matched by glob - - - **`max_objects_matched`** - _[integer]_ - Maximum number of objects matched by glob - - - **`max_objects_listed`** - _[integer]_ - Maximum number of objects listed in glob - - - **`page_size`** - _[integer]_ - Page size for glob listing - -### `batch_size` - -_[string]_ - Size of a batch (e.g., '100MB') - -## Additional properties when `connector` is `local_file` or [named connector](./connector.md#name) of local_file - -### `path` - -_[string]_ - Path to the data source. - -### `format` - -_[string]_ - Format of the data source (e.g., csv, json, parquet). - -## Additional properties when `connector` is `redshift` or [named connector](./connector.md#name) of redshift - -### `output_location` - -_[string]_ - S3 location where query results are stored. - -### `workgroup` - -_[string]_ - Redshift Serverless workgroup to use. - -### `database` - -_[string]_ - Name of the Redshift database. - -### `cluster_identifier` - -_[string]_ - Identifier of the Redshift cluster. - -### `role_arn` - -_[string]_ - ARN of the IAM role to assume for Redshift access. - -### `region` - -_[string]_ - AWS region of the Redshift deployment. - -## Additional properties when `connector` is `s3` or [named connector](./connector.md#name) of s3 - -### `region` - -_[string]_ - AWS region - -### `endpoint` - -_[string]_ - AWS Endpoint - -### `path` - -_[string]_ - Path to the source - -### `uri` - -_[string]_ - Source URI - -### `extract` - -_[object]_ - key-value pairs for extraction settings - -### `glob` - -_[object]_ - Settings related to glob file matching. - - - **`max_total_size`** - _[integer]_ - Maximum total size (in bytes) matched by glob - - - **`max_objects_matched`** - _[integer]_ - Maximum number of objects matched by glob - - - **`max_objects_listed`** - _[integer]_ - Maximum number of objects listed in glob - - - **`page_size`** - _[integer]_ - Page size for glob listing - -### `batch_size` - -_[string]_ - Size of a batch (e.g., '100MB') - -## Additional properties when `connector` is `salesforce` or [named connector](./connector.md#name) of salesforce - -### `soql` - -_[string]_ - SOQL query to execute against the Salesforce instance. - -### `sobject` - -_[string]_ - Salesforce object (e.g., Account, Contact) targeted by the query. - -### `queryAll` - -_[boolean]_ - Whether to include deleted and archived records in the query (uses queryAll API). \ No newline at end of file diff --git a/docs/docs/hidden/yaml/models.md b/docs/docs/hidden/yaml/models.md new file mode 100644 index 00000000000..14b1b2e9fe1 --- /dev/null +++ b/docs/docs/hidden/yaml/models.md @@ -0,0 +1,52 @@ +--- +note: GENERATED. DO NOT EDIT. +title: Model SQL +sidebar_position: 33 +--- + +When using Rill Developer, data transformations are powered by DuckDB and their dialect of SQL. Under the hood, by default, data models are created as views in DuckDB. Please check our modeling page and DuckDB documentation for more details about how to construct and write your model SQL syntax. + +In your Rill project directory, you can also create a `.sql` file containing an appropriate DuckDB `SELECT` statement, most commonly within the default `models` directory, to represent a model (or set of SQL transformations). Rill will automatically detect and parse the model next time you run `rill start`. + + ### Annotating your models with properties + In most cases, objects are represented in Rill as YAML files. Models are unique in that any model.sql file can be considered a model resource in Rill, representing a SQL transformation that you would like to inform using a set of inputs and outputting a view or table (depending on the materialization type). For most other resources, available properties can be set directly via the corresponding YAML file. In the case of a model SQL file though, configurable properties should be set by annotating the top of the file using the following syntax: + ```sql + -- @property: value + ``` + We will cover different available configurable properties in the below sections. + + +## Properties + +### `type` + +_[string]_ - By default, any new model that is created in a Rill project will populate a corresponding .sql file representing the model. Similarly, a .sql file that is directly created in the project directory will also be automatically assumed by Rill to be a model by default. Therefore, it is not necessary to annotate the model resource with the type property. + +For consistency or documentation purposes, if you'd like to annotate your model resource as well with the type property, you can do so by adding the following to the top of your model_name.sql: +```sql +-- @type: model +``` + _(required)_ + +### `materialize` + +_[boolean]_ - As mentioned, models will be materialized in DuckDB as views by default. However, you can choose to materialize them as tables instead of views. To do this, you can add the following annotation to the top of your model SQL file: +```sql +-- @materialize: true +``` + +Alternatively, it is possible to set it as a project-wide default as well that your models inherit via your rill.yaml file: +```yaml +models: + materialize: true +``` + +:::info To materialize or not to materialize? + +There are both pros and cons to materializing your models. +- Pros can include improved performance for downstream models and dashboards, especially with the SQL is complex and/or the data size is large. We generally recommend _materializing_ final models that power dashboards. +- Cons can include a degraded keystroke-by-keystroke modeling experience or for specific edge cases, such as when using cross joins. + +If unsure, we would generally recommend leaving the defaults and/or reaching out for further guidance! +::: + \ No newline at end of file diff --git a/docs/docs/hidden/yaml/project.md b/docs/docs/hidden/yaml/rillyaml.md similarity index 99% rename from docs/docs/hidden/yaml/project.md rename to docs/docs/hidden/yaml/rillyaml.md index 8a7504e1b77..ee3e3fe93db 100644 --- a/docs/docs/hidden/yaml/project.md +++ b/docs/docs/hidden/yaml/rillyaml.md @@ -1,7 +1,7 @@ --- note: GENERATED. DO NOT EDIT. title: Project YAML -sidebar_position: 39 +sidebar_position: 41 --- The `rill.yaml` file contains metadata about your project. diff --git a/docs/docs/hidden/yaml/sources.md b/docs/docs/hidden/yaml/sources.md new file mode 100644 index 00000000000..0c39caf279e --- /dev/null +++ b/docs/docs/hidden/yaml/sources.md @@ -0,0 +1,95 @@ +--- +note: GENERATED. DO NOT EDIT. +title: Source YAML +sidebar_position: 32 +--- + +:::warning Deprecated Feature +**Sources have been deprecated** and are now considered "source models." While sources remain backward compatible, we recommend migrating to the new source model format for access to the latest features and improvements. + +**Next steps:** +- Continue using sources if needed (backward compatible) +- Migrate to source models via the `type:model` parameter for existing projects +- See our [model YAML reference](advanced-models) for current documentation and best practices +::: + +Source YAML files define data sources that can be imported into Rill projects. + + +## Properties + +### `type` + +_[string]_ - Refers to the resource type and must be `model` _(required)_ + +### `connector` + +_[string]_ - Refers to the connector type for the source, see [connectors](/connect) for more information _(required)_ + +### `uri` + +_[string]_ - Refers to the URI of the remote connector you are using for the source. Rill also supports glob patterns as part of the URI for S3 and GCS (required for type: http, s3, gcs). + +- `s3://your-org/bucket/file.parquet` — the s3 URI of your file +- `gs://your-org/bucket/file.parquet` — the gsutil URI of your file +- `https://data.example.org/path/to/file.parquet` — the web address of your file + + +### `path` + +_[string]_ - Refers to the local path of the connector you are using for the source + +### `sql` + +_[string]_ - Sets the SQL query to extract data from a SQL source + +### `region` + +_[string]_ - Sets the cloud region of the S3 bucket or Athena + +### `endpoint` + +_[string]_ - Overrides the S3 endpoint to connect to + +### `output_location` + +_[string]_ - Sets the query output location and result files in Athena + +### `workgroup` + +_[string]_ - Sets a workgroup for Athena connector + +### `project_id` + +_[string]_ - Sets a project id to be used to run BigQuery jobs + +### `timeout` + +_[string]_ - The maximum time to wait for source ingestion + +### `refresh` + +_[object]_ - Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying source data (optional). +- **cron** - a cron schedule expression, which should be encapsulated in single quotes, e.g. `* * * * *` (optional) +- **every** - a Go duration string, such as `24h` (optional) + + + - **`cron`** - _[string]_ - A cron schedule expression, which should be encapsulated in single quotes, e.g. `* * * * *` + + - **`every`** - _[string]_ - A Go duration string, such as `24h` + +### `db` + +_[string]_ - Sets the database for motherduck connections and/or the path to the DuckDB/SQLite db file + +### `database_url` + +_[string]_ - Postgres connection string that should be used + +### `duckdb` + +_[object]_ - Specifies the raw parameters to inject into the DuckDB read_csv, read_json or read_parquet statement + +### `dsn` + +_[string]_ - Used to set the Snowflake connection string \ No newline at end of file diff --git a/docs/docs/hidden/yaml/theme.md b/docs/docs/hidden/yaml/theme.md deleted file mode 100644 index 35d35542419..00000000000 --- a/docs/docs/hidden/yaml/theme.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -note: GENERATED. DO NOT EDIT. -title: Theme YAML -sidebar_position: 39 ---- - -In your Rill project directory, create a `.yaml` file in any directory containing `type: theme`. Rill will automatically ingest the theme next time you run `rill start` or deploy to Rill Cloud. - -To apply that theme to a dashboard, add `default_theme: ` to the yaml file for that dashboard. Alternatively, you can add this to the end of the URL in your browser: `?theme=` - - -## Properties - -### `type` - -_[string]_ - Refers to the resource type and must be `theme` _(required)_ - -### `colors` - -_[object]_ - Used to override the dashboard colors. Either primary or secondary color must be provided. _(required)_ - - - **`primary`** - _[string]_ - Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). - - - **`secondary`** - _[string]_ - Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. - -## Common Properties - -### `name` - -_[string]_ - Name is usually inferred from the filename, but can be specified manually. - -### `refs` - -_[array of string]_ - List of resource references - -### `dev` - -_[object]_ - Overrides any properties in development environment. - -### `prod` - -_[object]_ - Overrides any properties in production environment. - -## Examples - -```yaml -# Example: You can copy this directly into your .yaml file -type: theme -colors: - primary: plum - secondary: violet -``` \ No newline at end of file diff --git a/docs/docs/hidden/yaml/themes.md b/docs/docs/hidden/yaml/themes.md index 35775e5b152..664037f8393 100644 --- a/docs/docs/hidden/yaml/themes.md +++ b/docs/docs/hidden/yaml/themes.md @@ -1,7 +1,7 @@ --- note: GENERATED. DO NOT EDIT. -title: Themes YAML -sidebar_position: 38 +title: Theme YAML +sidebar_position: 40 --- In your Rill project directory, create a `.yaml` file in any directory containing `type: theme`. Rill will automatically ingest the theme next time you run `rill start` or deploy to Rill Cloud. diff --git a/runtime/parser/schema/advanced-models.schema.yaml b/runtime/parser/schema/advanced-models.schema.yaml index d666994fc18..1690611fb44 100644 --- a/runtime/parser/schema/advanced-models.schema.yaml +++ b/runtime/parser/schema/advanced-models.schema.yaml @@ -1,6 +1,6 @@ $schema: 'http://json-schema.org/draft-07/schema#' $id: advanced-models.schema.yaml -title: Advanced Models YAML +title: Models YAML type: object allOf: - title: Properties diff --git a/runtime/parser/schema/alerts.schema.yaml b/runtime/parser/schema/alerts.schema.yaml index 5a7a80b5a25..cd2158fd735 100644 --- a/runtime/parser/schema/alerts.schema.yaml +++ b/runtime/parser/schema/alerts.schema.yaml @@ -1,6 +1,6 @@ $schema: 'http://json-schema.org/draft-07/schema#' $id: alerts.schema.yaml -title: Alerts YAML +title: Alert YAML type: object description: Along with alertings at the dashboard level and can be created via the UI, there might be more extensive alerting that you might want to develop and can be done so the an alert.yaml. When creating an alert via a YAML file, you'll see this denoted in the UI as `Created through code`. examples: diff --git a/runtime/parser/schema/apis.schema.yaml b/runtime/parser/schema/apis.schema.yaml index b978ef3f77e..583d0d84ef4 100644 --- a/runtime/parser/schema/apis.schema.yaml +++ b/runtime/parser/schema/apis.schema.yaml @@ -1,6 +1,6 @@ $schema: 'http://json-schema.org/draft-07/schema#' $id: apis.schema.yaml -title: APIs YAML +title: API YAML type: object description: In your Rill project directory, create a new file name `.yaml` in the `apis` directory containing a custom API definition. See comprehensive documentation on how to define and use [custom APIs](/integrate/custom-apis/index.md) examples: diff --git a/runtime/parser/schema/canvas-dashboards.schema.yaml b/runtime/parser/schema/canvas-dashboards.schema.yaml index 523d612f71f..aec7628bb8e 100644 --- a/runtime/parser/schema/canvas-dashboards.schema.yaml +++ b/runtime/parser/schema/canvas-dashboards.schema.yaml @@ -1,6 +1,6 @@ $schema: 'http://json-schema.org/draft-07/schema#' $id: canvas-dashboards.schema.yaml -title: Canvas Dashboards YAML +title: Canvas Dashboard YAML type: object description: In your Rill project directory, create a canvas dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. allOf: diff --git a/runtime/parser/schema/connectors.schema.yaml b/runtime/parser/schema/connectors.schema.yaml index 2372acbc029..c25b8a0c70f 100644 --- a/runtime/parser/schema/connectors.schema.yaml +++ b/runtime/parser/schema/connectors.schema.yaml @@ -1,6 +1,6 @@ $schema: 'http://json-schema.org/draft-07/schema#' $id: connectors.schema.yaml -title: Connectors YAML +title: Connector YAML type: object description: | Connector YAML files define how Rill connects to external data sources and OLAP engines. Each connector specifies a driver type and its required connection parameters. diff --git a/runtime/parser/schema/explore-dashboards.schema.yaml b/runtime/parser/schema/explore-dashboards.schema.yaml index 1eabb5cc500..42a382359f1 100644 --- a/runtime/parser/schema/explore-dashboards.schema.yaml +++ b/runtime/parser/schema/explore-dashboards.schema.yaml @@ -1,6 +1,6 @@ $schema: 'http://json-schema.org/draft-07/schema#' $id: explore-dashboards.schema.yaml -title: Explore Dashboards YAML +title: Explore Dashboard YAML type: object description: In your Rill project directory, create a explore dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. allOf: diff --git a/runtime/parser/schema/metrics_views.schema.yaml b/runtime/parser/schema/metrics_views.schema.yaml index b182c4b51f5..09eea4243a4 100644 --- a/runtime/parser/schema/metrics_views.schema.yaml +++ b/runtime/parser/schema/metrics_views.schema.yaml @@ -1,6 +1,6 @@ $schema: 'http://json-schema.org/draft-07/schema#' -$id: metrics_views.schema.yaml -title: Metrics Views YAML +$id: metrics-views.schema.yaml +title: Metrics View YAML type: object description: In your Rill project directory, create a metrics view, `.yaml`, file in the `metrics` directory. Rill will ingest the metric view definition next time you run `rill start`. allOf: diff --git a/runtime/parser/schema/models.schema.yaml b/runtime/parser/schema/models.schema.yaml new file mode 100644 index 00000000000..81ebabc7a43 --- /dev/null +++ b/runtime/parser/schema/models.schema.yaml @@ -0,0 +1,54 @@ +$schema: 'http://json-schema.org/draft-07/schema#' +$id: models.schema.yaml +title: Model SQL +type: object +description: | + When using Rill Developer, data transformations are powered by DuckDB and their dialect of SQL. Under the hood, by default, data models are created as views in DuckDB. Please check our modeling page and DuckDB documentation for more details about how to construct and write your model SQL syntax. + + In your Rill project directory, you can also create a `.sql` file containing an appropriate DuckDB `SELECT` statement, most commonly within the default `models` directory, to represent a model (or set of SQL transformations). Rill will automatically detect and parse the model next time you run `rill start`. + + ### Annotating your models with properties + In most cases, objects are represented in Rill as YAML files. Models are unique in that any model.sql file can be considered a model resource in Rill, representing a SQL transformation that you would like to inform using a set of inputs and outputting a view or table (depending on the materialization type). For most other resources, available properties can be set directly via the corresponding YAML file. In the case of a model SQL file though, configurable properties should be set by annotating the top of the file using the following syntax: + ```sql + -- @property: value + ``` + We will cover different available configurable properties in the below sections. + +allOf: + - title: Properties + type: object + properties: + type: + type: string + const: model + description: | + By default, any new model that is created in a Rill project will populate a corresponding .sql file representing the model. Similarly, a .sql file that is directly created in the project directory will also be automatically assumed by Rill to be a model by default. Therefore, it is not necessary to annotate the model resource with the type property. + + For consistency or documentation purposes, if you'd like to annotate your model resource as well with the type property, you can do so by adding the following to the top of your model_name.sql: + ```sql + -- @type: model + ``` + materialize: + type: boolean + description: | + As mentioned, models will be materialized in DuckDB as views by default. However, you can choose to materialize them as tables instead of views. To do this, you can add the following annotation to the top of your model SQL file: + ```sql + -- @materialize: true + ``` + + Alternatively, it is possible to set it as a project-wide default as well that your models inherit via your rill.yaml file: + ```yaml + models: + materialize: true + ``` + + :::info To materialize or not to materialize? + + There are both pros and cons to materializing your models. + - Pros can include improved performance for downstream models and dashboards, especially with the SQL is complex and/or the data size is large. We generally recommend _materializing_ final models that power dashboards. + - Cons can include a degraded keystroke-by-keystroke modeling experience or for specific edge cases, such as when using cross joins. + + If unsure, we would generally recommend leaving the defaults and/or reaching out for further guidance! + ::: + required: + - type \ No newline at end of file diff --git a/runtime/parser/schema/project.schema.yaml b/runtime/parser/schema/project.schema.yaml index debcbc6203e..fa2fb766b03 100644 --- a/runtime/parser/schema/project.schema.yaml +++ b/runtime/parser/schema/project.schema.yaml @@ -21,14 +21,18 @@ description: | ::: oneOf: - - $ref: 'alerts.schema.yaml#' - - $ref: 'apis.schema.yaml#' + - $ref: 'connectors.schema.yaml#' + - $ref: 'sources.schema.yaml#' + - $ref: 'models.schema.yaml#' + - $ref: 'advanced-models.schema.yaml#' + - $ref: 'metrics_views.schema.yaml#' - $ref: 'canvas-dashboards.schema.yaml#' # - $ref: 'component.schema.yaml#' - - $ref: 'connectors.schema.yaml#' - $ref: 'explore-dashboards.schema.yaml#' - - $ref: 'metrics_views.schema.yaml#' - - $ref: 'advanced-models.schema.yaml#' + + + - $ref: 'alerts.schema.yaml#' + - $ref: 'apis.schema.yaml#' - $ref: 'themes.schema.yaml#' definitions: common_properties: diff --git a/runtime/parser/schema/sources.schema.yaml b/runtime/parser/schema/sources.schema.yaml new file mode 100644 index 00000000000..5ab4ae2e672 --- /dev/null +++ b/runtime/parser/schema/sources.schema.yaml @@ -0,0 +1,100 @@ +$schema: 'http://json-schema.org/draft-07/schema#' +$id: sources.schema.yaml +title: Source YAML +type: object +description: | + :::warning Deprecated Feature + **Sources have been deprecated** and are now considered "source models." While sources remain backward compatible, we recommend migrating to the new source model format for access to the latest features and improvements. + + **Next steps:** + - Continue using sources if needed (backward compatible) + - Migrate to source models via the `type:model` parameter for existing projects + - See our [model YAML reference](advanced-models) for current documentation and best practices + ::: + + Source YAML files define data sources that can be imported into Rill projects. +allOf: + - title: Properties + type: object + properties: + type: + type: string + const: model + description: Refers to the resource type and must be `model` + connector: + type: string + description: Refers to the connector type for the source, see [connectors](/connect) for more information + enum: + - https + - s3 + - gcs + - local_file + - motherduck + - athena + - redshift + - postgres + - sqlite + - snowflake + - bigquery + - duckdb + uri: + type: string + description: | + Refers to the URI of the remote connector you are using for the source. Rill also supports glob patterns as part of the URI for S3 and GCS (required for type: http, s3, gcs). + + - `s3://your-org/bucket/file.parquet` — the s3 URI of your file + - `gs://your-org/bucket/file.parquet` — the gsutil URI of your file + - `https://data.example.org/path/to/file.parquet` — the web address of your file + path: + type: string + description: Refers to the local path of the connector you are using for the source + sql: + type: string + description: Sets the SQL query to extract data from a SQL source + region: + type: string + description: Sets the cloud region of the S3 bucket or Athena + endpoint: + type: string + description: Overrides the S3 endpoint to connect to + output_location: + type: string + description: Sets the query output location and result files in Athena + workgroup: + type: string + description: Sets a workgroup for Athena connector + project_id: + type: string + description: Sets a project id to be used to run BigQuery jobs + timeout: + type: string + description: The maximum time to wait for source ingestion + refresh: + type: object + description: | + Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying source data (optional). + - **cron** - a cron schedule expression, which should be encapsulated in single quotes, e.g. `* * * * *` (optional) + - **every** - a Go duration string, such as `24h` (optional) + properties: + cron: + type: string + description: A cron schedule expression, which should be encapsulated in single quotes, e.g. `* * * * *` + every: + type: string + description: A Go duration string, such as `24h` + db: + type: string + description: Sets the database for motherduck connections and/or the path to the DuckDB/SQLite db file + database_url: + type: string + description: Postgres connection string that should be used + duckdb: + type: object + description: Specifies the raw parameters to inject into the DuckDB read_csv, read_json or read_parquet statement + additionalProperties: true + dsn: + type: string + description: Used to set the Snowflake connection string + required: + - type + - connector \ No newline at end of file diff --git a/runtime/parser/schema/themes.schema.yaml b/runtime/parser/schema/themes.schema.yaml index 12e5d07d2b8..f1b8788b925 100644 --- a/runtime/parser/schema/themes.schema.yaml +++ b/runtime/parser/schema/themes.schema.yaml @@ -1,6 +1,6 @@ $schema: 'http://json-schema.org/draft-07/schema#' $id: themes.schema.yaml -title: Themes YAML +title: Theme YAML type: object description: | In your Rill project directory, create a `.yaml` file in any directory containing `type: theme`. Rill will automatically ingest the theme next time you run `rill start` or deploy to Rill Cloud. From 6ba0dcd3fe708061b809abfacdc2cb5e5887da05 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 4 Aug 2025 16:49:48 -0600 Subject: [PATCH 13/32] nit, fix links --- cli/cmd/docs/generate_project.go | 2 +- docs/docs/hidden/yaml/advanced-models.md | 9 ++++----- docs/docs/hidden/yaml/connectors.md | 2 -- docs/docs/hidden/yaml/sources.md | 2 +- .../parser/schema/advanced-models.schema.yaml | 18 +++++++----------- runtime/parser/schema/sources.schema.yaml | 2 +- 6 files changed, 14 insertions(+), 21 deletions(-) diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index 1e1072a8411..0c2261032e4 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -504,7 +504,7 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req doc.WriteString("- [**HTTPS**](#https) - Public files via HTTP/HTTPS\n") doc.WriteString("- [**Salesforce**](#salesforce) - Salesforce data\n") doc.WriteString("- [**Slack**](#slack) - Slack data\n") - doc.WriteString("- [**Local File**](#local_file) - Local file system\n\n") + // doc.WriteString("- [**Local File**](#local_file) - Local file system\n\n") doc.WriteString(":::warning Security Recommendation\n") doc.WriteString("For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/build/credentials/) for complete setup instructions.\n") diff --git a/docs/docs/hidden/yaml/advanced-models.md b/docs/docs/hidden/yaml/advanced-models.md index 037e58bf701..a5ef5462a14 100644 --- a/docs/docs/hidden/yaml/advanced-models.md +++ b/docs/docs/hidden/yaml/advanced-models.md @@ -26,12 +26,11 @@ _[object]_ - Specifies the refresh schedule that Rill should follow to re-ingest ### `connector` -_[object]_ - Connector YAML files define how Rill connects to external data sources and OLAP engines. Each connector specifies a driver type and its required connection parameters. - +_[string]_ - Refers to the resource type and must be `connector` - - **`type`** - _[string]_ - Refers to the resource type and must be `connector` _(required)_ +### `driver` - - **`driver`** - _[string]_ - The type of connector, see [available connectors](#available-connector-types) (required) _(required)_ +_[string]_ - The type of connector, see [available connectors](../yaml/connectors#available-connector-types) (required) ### `sql` @@ -417,7 +416,7 @@ _[object]_ - Settings related to glob file matching. _[string]_ - Size of a batch (e.g., '100MB') -## Additional properties when `connector` is [`salesforce`](./connector#salesforce) +## Additional properties when `connector` is [`salesforce`](./connectors#salesforce) ### `soql` diff --git a/docs/docs/hidden/yaml/connectors.md b/docs/docs/hidden/yaml/connectors.md index 783425659c6..8f996e30376 100644 --- a/docs/docs/hidden/yaml/connectors.md +++ b/docs/docs/hidden/yaml/connectors.md @@ -43,8 +43,6 @@ Choose from the following connector types based on your data source: - [**HTTPS**](#https) - Public files via HTTP/HTTPS - [**Salesforce**](#salesforce) - Salesforce data - [**Slack**](#slack) - Slack data -- [**Local File**](#local_file) - Local file system - :::warning Security Recommendation For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/build/credentials/) for complete setup instructions. ::: diff --git a/docs/docs/hidden/yaml/sources.md b/docs/docs/hidden/yaml/sources.md index 0c39caf279e..b3201ed7c34 100644 --- a/docs/docs/hidden/yaml/sources.md +++ b/docs/docs/hidden/yaml/sources.md @@ -24,7 +24,7 @@ _[string]_ - Refers to the resource type and must be `model` _(required)_ ### `connector` -_[string]_ - Refers to the connector type for the source, see [connectors](/connect) for more information _(required)_ +_[string]_ - Refers to the connector type for the source, see [connectors](/build/connect/) for more information _(required)_ ### `uri` diff --git a/runtime/parser/schema/advanced-models.schema.yaml b/runtime/parser/schema/advanced-models.schema.yaml index 1690611fb44..3bce9ec4e73 100644 --- a/runtime/parser/schema/advanced-models.schema.yaml +++ b/runtime/parser/schema/advanced-models.schema.yaml @@ -14,7 +14,12 @@ allOf: $ref: '#/definitions/schedule_properties' description: Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying model data connector: - $ref: 'connectors.schema.yaml#' + type: string + const: connector + description: Refers to the resource type and must be `connector` + driver: + type: string + description: The type of connector, see [available connectors](../yaml/connectors#available-connector-types) (required) sql: type: string description: Raw SQL query to run against source @@ -194,15 +199,6 @@ allOf: - connector then: $ref: '#/definitions/model/definitions/gcs' - # - if: - # title: Additional properties when `connector` is [`local_file`](./connectors#local_file) - # properties: - # connector: - # const: local_file - # required: - # - connector - # then: - # $ref: '#/definitions/model/definitions/local_file' - if: title: Additional properties when `connector` is [`redshift`](./connectors#redshift) properties: @@ -222,7 +218,7 @@ allOf: then: $ref: '#/definitions/model/definitions/s3' - if: - title: Additional properties when `connector` is [`salesforce`](./connector#salesforce) + title: Additional properties when `connector` is [`salesforce`](./connectors#salesforce) properties: connector: const: salesforce diff --git a/runtime/parser/schema/sources.schema.yaml b/runtime/parser/schema/sources.schema.yaml index 5ab4ae2e672..e6a6c7a47c3 100644 --- a/runtime/parser/schema/sources.schema.yaml +++ b/runtime/parser/schema/sources.schema.yaml @@ -23,7 +23,7 @@ allOf: description: Refers to the resource type and must be `model` connector: type: string - description: Refers to the connector type for the source, see [connectors](/connect) for more information + description: Refers to the connector type for the source, see [connectors](/build/connect/) for more information enum: - https - s3 From 4363ffffd115a58a876674057d0845231d71eedc Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Tue, 5 Aug 2025 14:09:34 -0600 Subject: [PATCH 14/32] comparing with current YAML changes --- cli/cmd/docs/generate_project.go | 73 +------- docs/docs/hidden/yaml/advanced-models.md | 120 ++++--------- docs/docs/hidden/yaml/alerts.md | 76 ++------ docs/docs/hidden/yaml/apis.md | 53 +----- docs/docs/hidden/yaml/canvas-dashboards.md | 102 ++++------- docs/docs/hidden/yaml/connectors.md | 22 +-- docs/docs/hidden/yaml/explore-dashboards.md | 166 +++--------------- docs/docs/hidden/yaml/metrics-views.md | 50 +++++- docs/docs/hidden/yaml/models.md | 2 +- docs/docs/hidden/yaml/rillyaml.md | 4 + docs/docs/hidden/yaml/sources.md | 4 +- .../parser/schema/advanced-models.schema.yaml | 18 ++ runtime/parser/schema/apis.schema.yaml | 2 + .../schema/canvas-dashboards.schema.yaml | 117 ++++++------ runtime/parser/schema/connectors.schema.yaml | 36 ++++ .../schema/explore-dashboards.schema.yaml | 23 ++- .../parser/schema/metrics_views.schema.yaml | 16 +- runtime/parser/schema/models.schema.yaml | 2 - runtime/parser/schema/rillyaml.schema.yaml | 5 + runtime/parser/schema/sources.schema.yaml | 4 +- 20 files changed, 328 insertions(+), 567 deletions(-) diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index 0c2261032e4..a69793c04b7 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -424,29 +424,17 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req if len(oneOf.Content) == 1 { doc.WriteString(generateDoc(sidebarPosition, level, oneOf.Content[0], indent, getRequiredMapFromNode(oneOf.Content[0]))) } else { + // Remove the summary list to avoid duplication with detailed sections if level == 1 { doc.WriteString("\n\n## One of Properties Options") - for _, item := range oneOf.Content { - title := getScalarValue(item, "title") - if title != "" { - anchor := strings.ToLower(strings.ReplaceAll(title, " ", "-")) - doc.WriteString(fmt.Sprintf("\n- [%s](#%s)", title, anchor)) - } + } - } + - for i, item := range oneOf.Content { + for _, item := range oneOf.Content { if hasType(item) || hasProperties(item) || hasCombinators(item) { - title := getScalarValue(item, "title") - if title != "" { - doc.WriteString(fmt.Sprintf("\n\n#### Option %d: %s", i+1, title)) - } else { - doc.WriteString(fmt.Sprintf("\n\n#### Option %d", i+1)) - } - doc.WriteString(fmt.Sprintf("\n\n**Type:** %s\n\n**Description:** %s", - getPrintableType(item), - getPrintableDescription(item, indent, "(no description)"))) - doc.WriteString(generateDoc(sidebarPosition, level, item, indent+" ", getRequiredMapFromNode(item))) + + doc.WriteString(generateDoc(sidebarPosition, level, item, indent+" ", getRequiredMapFromNode(item))) } } } @@ -465,55 +453,6 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req // AllOf if allOf := getNodeForKey(node, "allOf"); allOf != nil && allOf.Kind == yaml.SequenceNode { - // Special handling for connector allOf - if isConnector && level == 1 { - doc.WriteString("\n\n## Available Connector Types\n\n") - doc.WriteString("Choose from the following connector types based on your data source:\n\n") - - // Find the oneOf section within allOf - for _, item := range allOf.Content { - oneOf := getNodeForKey(item, "oneOf") - if oneOf == nil || oneOf.Kind != yaml.SequenceNode { - continue - } - - doc.WriteString("### OLAP Engines\n\n") - doc.WriteString("- [**DuckDB**](#duckdb) - Embedded DuckDB engine (default)\n") - doc.WriteString("- [**ClickHouse**](#clickhouse) - ClickHouse analytical database\n") - doc.WriteString("- [**MotherDuck**](#motherduck) - MotherDuck cloud database\n") - doc.WriteString("- [**Druid**](#druid) - Apache Druid\n") - doc.WriteString("- [**Pinot**](#pinot) - Apache Pinot\n\n") - - doc.WriteString("### Data Warehouses\n\n") - doc.WriteString("- [**Snowflake**](#snowflake) - Snowflake data warehouse\n") - doc.WriteString("- [**BigQuery**](#bigquery) - Google BigQuery\n") - doc.WriteString("- [**Redshift**](#redshift) - Amazon Redshift\n") - doc.WriteString("- [**Athena**](#athena) - Amazon Athena\n\n") - - doc.WriteString("### Databases\n\n") - doc.WriteString("- [**PostgreSQL**](#postgres) - PostgreSQL databases\n") - doc.WriteString("- [**MySQL**](#mysql) - MySQL databases\n") - doc.WriteString("- [**SQLite**](#sqlite) - SQLite databases\n\n") - - doc.WriteString("### Cloud Storage\n\n") - doc.WriteString("- [**GCS**](#gcs) - Google Cloud Storage\n") - doc.WriteString("- [**S3**](#s3) - Amazon S3 storage\n") - doc.WriteString("- [**Azure**](#azure) - Azure Blob Storage\n\n") - - doc.WriteString("### Other\n\n") - doc.WriteString("- [**HTTPS**](#https) - Public files via HTTP/HTTPS\n") - doc.WriteString("- [**Salesforce**](#salesforce) - Salesforce data\n") - doc.WriteString("- [**Slack**](#slack) - Slack data\n") - // doc.WriteString("- [**Local File**](#local_file) - Local file system\n\n") - - doc.WriteString(":::warning Security Recommendation\n") - doc.WriteString("For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/build/credentials/) for complete setup instructions.\n") - doc.WriteString(":::\n\n") - - doc.WriteString("## Connector Details\n\n") - break - } - } for _, item := range allOf.Content { // Skip oneOf items for connectors since we handle them separately diff --git a/docs/docs/hidden/yaml/advanced-models.md b/docs/docs/hidden/yaml/advanced-models.md index a5ef5462a14..b909e356351 100644 --- a/docs/docs/hidden/yaml/advanced-models.md +++ b/docs/docs/hidden/yaml/advanced-models.md @@ -4,6 +4,22 @@ title: Models YAML sidebar_position: 34 --- +:::tip + +Both regular models and source models can use the Model YAML specification described on this page. While [SQL models](./models) are perfect for simple transformations, Model YAML files provide advanced capabilities for complex data processing scenarios. + +**When to use Model YAML:** +- **Partitions** - Optimize performance with data partitioning strategies +- **Incremental models** - Process only new or changed data efficiently +- **Pre/post execution hooks** - Run custom logic before or after model execution +- **Staging** - Create intermediate tables for complex transformations +- **Output configuration** - Define specific output formats and destinations + +Model YAML files give you fine-grained control over how your data is processed and transformed, making them ideal for production workloads and complex analytics pipelines. + +::: + + ## Properties ### `type` @@ -52,113 +68,53 @@ _[string]_ - Configure how changes to the model specifications are applied (opti _[oneOf]_ - Refers to the explicitly defined state of your model, cannot be used with partitions (optional) -#### Option 1: SQL Query - -**Type:** _[object]_ - -**Description:** Executes a raw SQL query against the project's data models. - - - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - - - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. - -#### Option 2: Metrics View Query - -**Type:** _[object]_ - -**Description:** Executes a SQL query that targets a defined metrics view. - - - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - -#### Option 3: Custom API Call - -**Type:** _[object]_ - -**Description:** Calls a custom API defined in the project to compute data. - - - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ + - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - - **`args`** - _[object]_ - Arguments to pass to the custom API. + - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. -#### Option 4: File Glob Query + - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ -**Type:** _[object]_ + - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ -**Description:** Uses a file-matching pattern (glob) to query data from a connector. + - **`args`** - _[object]_ - Arguments to pass to the custom API. - - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ + - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - - **option 1** - _[string]_ - A simple file path/glob pattern as a string. + - **option 1** - _[string]_ - A simple file path/glob pattern as a string. - - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. + - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. - - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. + - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. -#### Option 5: Resource Status Check + - **`resource_status`** - _[object]_ - Based on resource status _(required)_ -**Type:** _[object]_ - -**Description:** Uses the status of a resource as data. - - - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - - - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. + - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. ### `partitions` _[oneOf]_ - Refers to the how your data is partitioned, cannot be used with state. (optional) -#### Option 1: SQL Query - -**Type:** _[object]_ - -**Description:** Executes a raw SQL query against the project's data models. - - - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - - - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. - -#### Option 2: Metrics View Query - -**Type:** _[object]_ - -**Description:** Executes a SQL query that targets a defined metrics view. - - - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - -#### Option 3: Custom API Call - -**Type:** _[object]_ - -**Description:** Calls a custom API defined in the project to compute data. - - - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - - - **`args`** - _[object]_ - Arguments to pass to the custom API. - -#### Option 4: File Glob Query - -**Type:** _[object]_ + - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ -**Description:** Uses a file-matching pattern (glob) to query data from a connector. + - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. - - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ + - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - - **option 1** - _[string]_ - A simple file path/glob pattern as a string. + - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. + - **`args`** - _[object]_ - Arguments to pass to the custom API. - - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. + - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ -#### Option 5: Resource Status Check + - **option 1** - _[string]_ - A simple file path/glob pattern as a string. -**Type:** _[object]_ + - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. -**Description:** Uses the status of a resource as data. + - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. - - **`resource_status`** - _[object]_ - Based on resource status _(required)_ + - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. + - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. ### `materialize` diff --git a/docs/docs/hidden/yaml/alerts.md b/docs/docs/hidden/yaml/alerts.md index c405a8d7e95..882d0a10a56 100644 --- a/docs/docs/hidden/yaml/alerts.md +++ b/docs/docs/hidden/yaml/alerts.md @@ -52,85 +52,37 @@ _[string]_ - define the timeout of the alert in seconds (optional). _[oneOf]_ - Specifies one of the options to retrieve or compute the data used by alert _(required)_ -#### Option 1: SQL Query + - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ -**Type:** _[object]_ + - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. -**Description:** Executes a raw SQL query against the project's data models. + - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ + - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. + - **`args`** - _[object]_ - Arguments to pass to the custom API. -#### Option 2: Metrics View Query + - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ -**Type:** _[object]_ + - **option 1** - _[string]_ - A simple file path/glob pattern as a string. -**Description:** Executes a SQL query that targets a defined metrics view. + - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. - - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ + - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. -#### Option 3: Custom API Call + - **`resource_status`** - _[object]_ - Based on resource status _(required)_ -**Type:** _[object]_ - -**Description:** Calls a custom API defined in the project to compute data. - - - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - - - **`args`** - _[object]_ - Arguments to pass to the custom API. - -#### Option 4: File Glob Query - -**Type:** _[object]_ - -**Description:** Uses a file-matching pattern (glob) to query data from a connector. - - - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - - - **option 1** - _[string]_ - A simple file path/glob pattern as a string. - - - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. - - - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. - -#### Option 5: Resource Status Check - -**Type:** _[object]_ - -**Description:** Uses the status of a resource as data. - - - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - - - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. + - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. ### `for` _[oneOf]_ - Specifies how user identity or attributes should be evaluated for security policy enforcement. -#### Option 1 - -**Type:** _[object]_ - -**Description:** Specifies a unique user identifier for applying security policies. - - - **`user_id`** - _[string]_ - The unique user ID used to evaluate security policies. _(required)_ - -#### Option 2 - -**Type:** _[object]_ - -**Description:** Specifies a user's email address for applying security policies. - - - **`user_email`** - _[string]_ - The user's email address used to evaluate security policies. _(required)_ - -#### Option 3 - -**Type:** _[object]_ + - **`user_id`** - _[string]_ - The unique user ID used to evaluate security policies. _(required)_ -**Description:** Specifies a set of arbitrary user attributes for applying security policies. + - **`user_email`** - _[string]_ - The user's email address used to evaluate security policies. _(required)_ - - **`attributes`** - _[object]_ - A dictionary of user attributes used to evaluate security policies. _(required)_ + - **`attributes`** - _[object]_ - A dictionary of user attributes used to evaluate security policies. _(required)_ ### `on_recover` diff --git a/docs/docs/hidden/yaml/apis.md b/docs/docs/hidden/yaml/apis.md index 00d49177c94..8ad4b3051f8 100644 --- a/docs/docs/hidden/yaml/apis.md +++ b/docs/docs/hidden/yaml/apis.md @@ -30,18 +30,6 @@ _[object]_ - Defines security rules and access control policies for resources - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. -#### Option 1 - -**Type:** _[string]_ - -**Description:** SQL expression that evaluates to a boolean to determine access - -#### Option 2 - -**Type:** _[boolean]_ - -**Description:** Direct boolean value to allow or deny access - - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded @@ -83,17 +71,6 @@ _[object]_ - Defines security rules and access control policies for resources _[boolean]_ - Flag to control security inheritance ## One of Properties Options -- [SQL Query](#sql-query) -- [Metrics View Query](#metrics-view-query) -- [Custom API Call](#custom-api-call) -- [File Glob Query](#file-glob-query) -- [Resource Status Check](#resource-status-check) - -#### Option 1: SQL Query - -**Type:** _[object]_ - -**Description:** Executes a raw SQL query against the project's data models. ## SQL Query @@ -107,12 +84,6 @@ _[string]_ - Raw SQL query to run against existing models in the project. _(requ _[string]_ - specifies the connector to use when running SQL or glob queries. -#### Option 2: Metrics View Query - -**Type:** _[object]_ - -**Description:** Executes a SQL query that targets a defined metrics view. - ## Metrics View Query Executes a SQL query that targets a defined metrics view. @@ -121,12 +92,6 @@ Executes a SQL query that targets a defined metrics view. _[string]_ - SQL query that targets a metrics view in the project _(required)_ -#### Option 3: Custom API Call - -**Type:** _[object]_ - -**Description:** Calls a custom API defined in the project to compute data. - ## Custom API Call Calls a custom API defined in the project to compute data. @@ -139,12 +104,6 @@ _[string]_ - Name of a custom API defined in the project. _(required)_ _[object]_ - Arguments to pass to the custom API. -#### Option 4: File Glob Query - -**Type:** _[object]_ - -**Description:** Uses a file-matching pattern (glob) to query data from a connector. - ## File Glob Query Uses a file-matching pattern (glob) to query data from a connector. @@ -153,20 +112,14 @@ Uses a file-matching pattern (glob) to query data from a connector. _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - - **option 1** - _[string]_ - A simple file path/glob pattern as a string. + - **option 1** - _[string]_ - A simple file path/glob pattern as a string. - - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. + - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. ### `connector` _[string]_ - Specifies the connector to use with the glob input. -#### Option 5: Resource Status Check - -**Type:** _[object]_ - -**Description:** Uses the status of a resource as data. - ## Resource Status Check Uses the status of a resource as data. @@ -175,7 +128,7 @@ Uses the status of a resource as data. _[object]_ - Based on resource status _(required)_ - - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. + - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. ## Common Properties diff --git a/docs/docs/hidden/yaml/canvas-dashboards.md b/docs/docs/hidden/yaml/canvas-dashboards.md index 68238dbdd44..d0a0812bda0 100644 --- a/docs/docs/hidden/yaml/canvas-dashboards.md +++ b/docs/docs/hidden/yaml/canvas-dashboards.md @@ -20,6 +20,32 @@ _[string]_ - Refers to the display name for the canvas _[string]_ - Refers to the custom banner displayed at the header of an Canvas dashboard +### `rows` + +_[array of object]_ - Refers to all of the rows displayed on the Canvas _(required)_ + + - **`height`** - _[string]_ - Height of the row in px + + - **`items`** - _[array of object]_ - List of components to display in the row + + - **`component`** - _[string]_ - Name of the component to display. Each component type has its own set of properties. + Available component types: + + - **markdown** - Text component, uses markdown formatting + - **kpi_grid** - KPI component, similar to TDD in Rill Explore, display quick KPI charts + - **stacked_bar_normalized** - Bar chart normalized to 100% values + - **line_chart** - Normal Line chart + - **bar_chart** - Normal Bar chart + - **stacked_bar** - Stacked Bar chart + - **area_chart** - Line chart with area + - **image** - Provide a URL to embed into canvas dashboard + - **table** - Similar to Pivot table, add dimensions and measures to visualize your data + - **heatmap** - Heat Map chart to visualize distribution of data + - **donut_chart** - Donut or Pie chart to display sums of total + + + - **`width`** - _[string, integer]_ - Width of the component (can be a number or string with unit) + ### `max_width` _[integer]_ - Max width in pixels of the canvas @@ -34,25 +60,13 @@ _[integer]_ - Vertical gap in pixels of the canvas ### `theme` -_[oneOf]_ - Name of the theme to use. Only one of theme and embedded_theme can be set. - -#### Option 1 +_[oneOf]_ - Theme configuration. Can be either a string reference to an existing theme or an inline theme configuration object. -**Type:** _[string]_ + - **`colors`** - _[object]_ - Used to override the dashboard colors. Either primary or secondary color must be provided. -**Description:** Name of an existing theme to apply to the dashboard + - **`primary`** - _[string]_ - Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). -#### Option 2 - -**Type:** _[object]_ - -**Description:** Inline theme configuration. - - - **`colors`** - _[object]_ - Used to override the dashboard colors. Either primary or secondary color must be provided. - - - **`primary`** - _[string]_ - Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). - - - **`secondary`** - _[string]_ - Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. + - **`secondary`** - _[string]_ - Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. ### `allow_custom_time_range` @@ -62,37 +76,13 @@ _[boolean]_ - Defaults to true, when set to false it will hide the ability to se _[array of oneOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' -#### Option 1 - -**Type:** _[string]_ - -**Description:** a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection - -#### Option 2 - -**Type:** _[object]_ - -**Description:** Object containing time range and comparison configuration - - - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ + - **`range`** - _[string]_ - A valid ISO 8601 duration or one of the Rill ISO 8601 extensions for the selection _(required)_ - - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + - **`comparison_offsets`** - _[array of oneOf]_ - List of time comparison options for this time range selection (optional). Must be one of the Rill ISO 8601 extensions -#### Option 1 + - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) -**Type:** _[string]_ - -**Description:** Offset string only (range is inferred) - -#### Option 2 - -**Type:** _[object]_ - -**Description:** Object containing offset and range configuration for time comparison - - - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) - - - **`range`** - _[string]_ - Custom time range for comparison period + - **`range`** - _[string]_ - Custom time range for comparison period ### `time_zones` @@ -124,36 +114,12 @@ _[array of object]_ - Variables that can be used in the canvas - **`value`** - _[string, number, boolean, object, array]_ - Default value for the variable. Can be any valid JSON value type -### `rows` - -_[array of object]_ - Refers to all of the rows displayed on the Canvas _(required)_ - - - **`height`** - _[string]_ - Height of the row in px - - - **`items`** - _[array of object]_ - List of components to display in the row - - - **`component`** - _[string]_ - Name of the component to display - - - **`width`** - _[string, integer]_ - Width of the component (can be a number or string with unit) - ### `security` _[object]_ - Defines security rules and access control policies for resources - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. -#### Option 1 - -**Type:** _[string]_ - -**Description:** SQL expression that evaluates to a boolean to determine access - -#### Option 2 - -**Type:** _[boolean]_ - -**Description:** Direct boolean value to allow or deny access - - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded diff --git a/docs/docs/hidden/yaml/connectors.md b/docs/docs/hidden/yaml/connectors.md index 8f996e30376..5c7af185745 100644 --- a/docs/docs/hidden/yaml/connectors.md +++ b/docs/docs/hidden/yaml/connectors.md @@ -6,49 +6,41 @@ sidebar_position: 31 Connector YAML files define how Rill connects to external data sources and OLAP engines. Each connector specifies a driver type and its required connection parameters. - ## Available Connector Types -Choose from the following connector types based on your data source: - -### OLAP Engines - +### _OLAP Engines_ - [**DuckDB**](#duckdb) - Embedded DuckDB engine (default) - [**ClickHouse**](#clickhouse) - ClickHouse analytical database - [**MotherDuck**](#motherduck) - MotherDuck cloud database - [**Druid**](#druid) - Apache Druid - [**Pinot**](#pinot) - Apache Pinot -### Data Warehouses - +### _Data Warehouses_ - [**Snowflake**](#snowflake) - Snowflake data warehouse - [**BigQuery**](#bigquery) - Google BigQuery - [**Redshift**](#redshift) - Amazon Redshift - [**Athena**](#athena) - Amazon Athena -### Databases - +### _Databases_ - [**PostgreSQL**](#postgres) - PostgreSQL databases - [**MySQL**](#mysql) - MySQL databases - [**SQLite**](#sqlite) - SQLite databases -### Cloud Storage - +### _Cloud Storage_ - [**GCS**](#gcs) - Google Cloud Storage - [**S3**](#s3) - Amazon S3 storage - [**Azure**](#azure) - Azure Blob Storage -### Other - +### _Other_ - [**HTTPS**](#https) - Public files via HTTP/HTTPS - [**Salesforce**](#salesforce) - Salesforce data - [**Slack**](#slack) - Slack data + :::warning Security Recommendation For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/build/credentials/) for complete setup instructions. ::: -## Connector Details - +-- ## Properties diff --git a/docs/docs/hidden/yaml/explore-dashboards.md b/docs/docs/hidden/yaml/explore-dashboards.md index 428026f6b55..69cb970a301 100644 --- a/docs/docs/hidden/yaml/explore-dashboards.md +++ b/docs/docs/hidden/yaml/explore-dashboards.md @@ -16,6 +16,10 @@ _[string]_ - Refers to the resource type and must be `explore` _(required)_ _[string]_ - Refers to the display name for the explore dashboard +### `metrics_view` + +_[string]_ - Refers to the metrics view resource + ### `description` _[string]_ - Refers to the description of the explore dashboard @@ -24,123 +28,41 @@ _[string]_ - Refers to the description of the explore dashboard _[string]_ - Refers to the custom banner displayed at the header of an explore dashboard -### `metrics_view` - -_[string]_ - Refers to the metrics view resource - ### `dimensions` _[oneOf]_ - List of dimension names. Use '*' to select all dimensions (default) -#### Option 1 - -**Type:** _[string]_ - -**Description:** Wildcard(*) selector that includes all available fields in the selection - -#### Option 2 - -**Type:** _[array of string]_ - -**Description:** Explicit list of fields to include in the selection - -#### Option 3 - -**Type:** _[object]_ - -**Description:** Advanced matching using regex, DuckDB expression, or exclusion - - - **`regex`** - _[string]_ - Select dimensions using a regular expression + - **`regex`** - _[string]_ - Select dimensions using a regular expression - - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic + - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - - **`exclude`** - _[object]_ - Select all dimensions except those listed here + - **`exclude`** - _[object]_ - Select all dimensions except those listed here ### `measures` _[oneOf]_ - List of measure names. Use ''*'' to select all measures (default) -#### Option 1 - -**Type:** _[string]_ - -**Description:** Wildcard(*) selector that includes all available fields in the selection - -#### Option 2 - -**Type:** _[array of string]_ - -**Description:** Explicit list of fields to include in the selection - -#### Option 3 - -**Type:** _[object]_ - -**Description:** Advanced matching using regex, DuckDB expression, or exclusion - - - **`regex`** - _[string]_ - Select dimensions using a regular expression + - **`regex`** - _[string]_ - Select dimensions using a regular expression - - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic + - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - - **`exclude`** - _[object]_ - Select all dimensions except those listed here + - **`exclude`** - _[object]_ - Select all dimensions except those listed here ### `theme` _[oneOf]_ - Name of the theme to use. Only one of theme and embedded_theme can be set. -#### Option 1 - -**Type:** _[string]_ - -**Description:** Name of an existing theme to apply to the dashboard - -#### Option 2 - -**Type:** _[object]_ - -**Description:** Inline theme configuration. - - - **`colors`** - _[object]_ - Used to override the dashboard colors. Either primary or secondary color must be provided. - - - **`primary`** - _[string]_ - Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). - - - **`secondary`** - _[string]_ - Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. - ### `time_ranges` _[array of oneOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' -#### Option 1 - -**Type:** _[string]_ + - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ -**Description:** a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection + - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) -#### Option 2 + - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) -**Type:** _[object]_ - -**Description:** Object containing time range and comparison configuration - - - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ - - - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) - -#### Option 1 - -**Type:** _[string]_ - -**Description:** Offset string only (range is inferred) - -#### Option 2 - -**Type:** _[object]_ - -**Description:** Object containing offset and range configuration for time comparison - - - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) - - - **`range`** - _[string]_ - Custom time range for comparison period + - **`range`** - _[string]_ - Custom time range for comparison period ### `time_zones` @@ -160,55 +82,19 @@ _[object]_ - defines the defaults YAML struct - **`dimensions`** - _[oneOf]_ - Provides the default dimensions to load on viewing the dashboard -#### Option 1 + - **`regex`** - _[string]_ - Select dimensions using a regular expression -**Type:** _[string]_ + - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic -**Description:** Wildcard(*) selector that includes all available fields in the selection - -#### Option 2 - -**Type:** _[array of string]_ - -**Description:** Explicit list of fields to include in the selection - -#### Option 3 - -**Type:** _[object]_ - -**Description:** Advanced matching using regex, DuckDB expression, or exclusion - - - **`regex`** - _[string]_ - Select dimensions using a regular expression - - - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - - - **`exclude`** - _[object]_ - Select all dimensions except those listed here + - **`exclude`** - _[object]_ - Select all dimensions except those listed here - **`measures`** - _[oneOf]_ - Provides the default measures to load on viewing the dashboard -#### Option 1 - -**Type:** _[string]_ - -**Description:** Wildcard(*) selector that includes all available fields in the selection - -#### Option 2 - -**Type:** _[array of string]_ - -**Description:** Explicit list of fields to include in the selection - -#### Option 3 - -**Type:** _[object]_ - -**Description:** Advanced matching using regex, DuckDB expression, or exclusion + - **`regex`** - _[string]_ - Select dimensions using a regular expression - - **`regex`** - _[string]_ - Select dimensions using a regular expression + - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - - - **`exclude`** - _[object]_ - Select all dimensions except those listed here + - **`exclude`** - _[object]_ - Select all dimensions except those listed here - **`time_range`** - _[string]_ - Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) @@ -228,18 +114,6 @@ _[object]_ - Defines security rules and access control policies for resources - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. -#### Option 1 - -**Type:** _[string]_ - -**Description:** SQL expression that evaluates to a boolean to determine access - -#### Option 2 - -**Type:** _[boolean]_ - -**Description:** Direct boolean value to allow or deny access - - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded diff --git a/docs/docs/hidden/yaml/metrics-views.md b/docs/docs/hidden/yaml/metrics-views.md index 89b13eb08eb..e084d7185c4 100644 --- a/docs/docs/hidden/yaml/metrics-views.md +++ b/docs/docs/hidden/yaml/metrics-views.md @@ -8,9 +8,13 @@ In your Rill project directory, create a metrics view, `.yaml`, fi ## Properties +### `version` + +_[string]_ - The version of the metrics view schema _(required)_ + ### `type` -_[string]_ - Refers to the resource type and must be `metrics_view` +_[string]_ - Refers to the resource type and must be `metrics_view` _(required)_ ### `display_name` @@ -92,7 +96,7 @@ _[array of object]_ - Used to define the numeric aggregates of columns from your - **`expression`** - _[string]_ - a combination of operators and functions for aggregations - - **`window`** - _[anyOf]_ - A measure window can be defined as a keyword string (e.g. 'time' or 'all') or an object with detailed window configuration. + - **`window`** - _[anyOf]_ - A measure window can be defined as a keyword string (e.g. 'time' or 'all') or an object with detailed window configuration. For more information, see the [window functions](/build/metrics-view/advanced-expressions/windows) documentation. - **option 1** - _[string]_ - Shorthand: `time` or `true` means time-partitioned, `all` means non-partitioned. @@ -163,9 +167,47 @@ _[array of object]_ - Used to define the numeric aggregates of columns from your - **`treat_nulls_as`** - _[string]_ - used to configure what value to fill in for missing time buckets. This also works generally as COALESCING over non empty time buckets. -### `required` +### `security` + +_[object]_ - Defines security rules and access control policies for resources + + - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + + - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause + + - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded + + - **`if`** - _[string]_ - Expression to decide if the column should be included or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ + + - **`names`** - _[anyOf]_ - List of fields to include. Should match the name of one of the dashboard's dimensions or measures _(required)_ + + - **option 1** - _[array of string]_ - List of specific field names to include + + - **option 2** - _[string]_ - Wildcard '*' to include all fields + + - **`exclude`** - _[array of object]_ - List of dimension or measure names to exclude from the dashboard. If exclude is defined all other dimensions and measures are included + + - **`if`** - _[string]_ - Expression to decide if the column should be excluded or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ + + - **`names`** - _[anyOf]_ - List of fields to exclude. Should match the name of one of the dashboard's dimensions or measures _(required)_ + + - **option 1** - _[array of string]_ - List of specific field names to exclude + + - **option 2** - _[string]_ - Wildcard '*' to exclude all fields + + - **`rules`** - _[array of object]_ - List of detailed security rules that can be used to define complex access control policies + + - **`type`** - _[string]_ - Type of security rule - access (overall access), field_access (field-level access), or row_filter (row-level filtering) _(required)_ + + - **`action`** - _[string]_ - Whether to allow or deny access for this rule + + - **`if`** - _[string]_ - Conditional expression that determines when this rule applies. Must be a valid SQL expression that evaluates to a boolean + + - **`names`** - _[array of string]_ - List of field names this rule applies to (for field_access type rules) + + - **`all`** - _[boolean]_ - When true, applies the rule to all fields (for field_access type rules) -_[no type]_ - (no description) + - **`sql`** - _[string]_ - SQL expression for row filtering (for row_filter type rules) ## Common Properties diff --git a/docs/docs/hidden/yaml/models.md b/docs/docs/hidden/yaml/models.md index 14b1b2e9fe1..e87fd3aa0ba 100644 --- a/docs/docs/hidden/yaml/models.md +++ b/docs/docs/hidden/yaml/models.md @@ -26,7 +26,7 @@ For consistency or documentation purposes, if you'd like to annotate your model ```sql -- @type: model ``` - _(required)_ + ### `materialize` diff --git a/docs/docs/hidden/yaml/rillyaml.md b/docs/docs/hidden/yaml/rillyaml.md index ee3e3fe93db..bcb471d9f27 100644 --- a/docs/docs/hidden/yaml/rillyaml.md +++ b/docs/docs/hidden/yaml/rillyaml.md @@ -189,6 +189,10 @@ mock_users: groups: - partners - email: anon@unknown.com + - email: embed@rilldata.com + name: embed + custom_variable_1: Value_1 + custom_variable_2: Value_2 ``` ## Common Properties diff --git a/docs/docs/hidden/yaml/sources.md b/docs/docs/hidden/yaml/sources.md index b3201ed7c34..3d5315350e0 100644 --- a/docs/docs/hidden/yaml/sources.md +++ b/docs/docs/hidden/yaml/sources.md @@ -13,8 +13,6 @@ sidebar_position: 32 - See our [model YAML reference](advanced-models) for current documentation and best practices ::: -Source YAML files define data sources that can be imported into Rill projects. - ## Properties @@ -24,7 +22,7 @@ _[string]_ - Refers to the resource type and must be `model` _(required)_ ### `connector` -_[string]_ - Refers to the connector type for the source, see [connectors](/build/connect/) for more information _(required)_ +_[string]_ - Refers to the connector type for the source, see [connectors](/reference/project-files/connectors) for more information _(required)_ ### `uri` diff --git a/runtime/parser/schema/advanced-models.schema.yaml b/runtime/parser/schema/advanced-models.schema.yaml index 3bce9ec4e73..f9557e53e36 100644 --- a/runtime/parser/schema/advanced-models.schema.yaml +++ b/runtime/parser/schema/advanced-models.schema.yaml @@ -2,6 +2,24 @@ $schema: 'http://json-schema.org/draft-07/schema#' $id: advanced-models.schema.yaml title: Models YAML type: object +description: | + :::tip + + Both regular models and source models can use the Model YAML specification described on this page. While [SQL models](./models) are perfect for simple transformations, Model YAML files provide advanced capabilities for complex data processing scenarios. + + **When to use Model YAML:** + - **Partitions** - Optimize performance with data partitioning strategies + - **Incremental models** - Process only new or changed data efficiently + - **Pre/post execution hooks** - Run custom logic before or after model execution + - **Staging** - Create intermediate tables for complex transformations + - **Output configuration** - Define specific output formats and destinations + + Model YAML files give you fine-grained control over how your data is processed and transformed, making them ideal for production workloads and complex analytics pipelines. + + ::: + + + allOf: - title: Properties type: object diff --git a/runtime/parser/schema/apis.schema.yaml b/runtime/parser/schema/apis.schema.yaml index 583d0d84ef4..7f142813906 100644 --- a/runtime/parser/schema/apis.schema.yaml +++ b/runtime/parser/schema/apis.schema.yaml @@ -120,6 +120,8 @@ definitions: additionalProperties: true required: - resource_status + + security_policy_properties: type: object description: Defines security rules and access control policies for resources diff --git a/runtime/parser/schema/canvas-dashboards.schema.yaml b/runtime/parser/schema/canvas-dashboards.schema.yaml index aec7628bb8e..a5fd240cc2e 100644 --- a/runtime/parser/schema/canvas-dashboards.schema.yaml +++ b/runtime/parser/schema/canvas-dashboards.schema.yaml @@ -17,6 +17,46 @@ allOf: banner: type: string description: Refers to the custom banner displayed at the header of an Canvas dashboard + rows: + type: array + description: Refers to all of the rows displayed on the Canvas + items: + type: object + properties: + height: + type: string + description: Height of the row in px + items: + type: array + description: List of components to display in the row + items: + type: object + properties: + component: + type: string + description: | + Name of the component to display. Each component type has its own set of properties. + Available component types: + + - **markdown** - Text component, uses markdown formatting + - **kpi_grid** - KPI component, similar to TDD in Rill Explore, display quick KPI charts + - **stacked_bar_normalized** - Bar chart normalized to 100% values + - **line_chart** - Normal Line chart + - **bar_chart** - Normal Bar chart + - **stacked_bar** - Stacked Bar chart + - **area_chart** - Line chart with area + - **image** - Provide a URL to embed into canvas dashboard + - **table** - Similar to Pivot table, add dimensions and measures to visualize your data + - **heatmap** - Heat Map chart to visualize distribution of data + - **donut_chart** - Donut or Pie chart to display sums of total + + width: + type: + - string + - integer + description: Width of the component (can be a number or string with unit) + additionalProperties: true + additionalProperties: false max_width: type: integer description: Max width in pixels of the canvas @@ -28,14 +68,20 @@ allOf: gap_y: type: integer description: Vertical gap in pixels of the canvas - minimum: 0 + minimum: theme: oneOf: - - type: string + - title: Existing theme + type: string description: Name of an existing theme to apply to the dashboard - - $ref: '#/definitions/theme/definitions/theme_properties' + - title: Inline theme + type: object description: Inline theme configuration. - description: Name of the theme to use. Only one of theme and embedded_theme can be set. + $ref: '#/definitions/theme' + + description: Theme configuration. Can be either a string reference to an existing theme or an inline theme configuration object. + + allow_custom_time_range: type: boolean description: Defaults to true, when set to false it will hide the ability to set a custom time range for the user. @@ -76,31 +122,6 @@ allOf: description: Variables that can be used in the canvas items: $ref: '#/definitions/component_variable_properties' - rows: - type: array - description: Refers to all of the rows displayed on the Canvas - items: - type: object - properties: - height: - type: string - description: Height of the row in px - items: - type: array - description: List of components to display in the row - items: - type: object - properties: - component: - type: string - description: Name of the component to display - width: - type: - - string - - integer - description: Width of the component (can be a number or string with unit) - additionalProperties: true - additionalProperties: false security: $ref: '#/definitions/security_policy_properties' description: Security rules to apply for access to the canvas @@ -112,16 +133,16 @@ definitions: explore_time_range_properties: oneOf: - type: string - description: a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection + description: A valid ISO 8601 duration or one of the Rill ISO 8601 extensions for the selection - type: object description: Object containing time range and comparison configuration properties: range: type: string - description: a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection + description: A valid ISO 8601 duration or one of the Rill ISO 8601 extensions for the selection comparison_offsets: type: array - description: list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + description: List of time comparison options for this time range selection (optional). Must be one of the Rill ISO 8601 extensions items: oneOf: - type: string @@ -276,22 +297,20 @@ definitions: type: object description: Overrides any properties in production environment. theme: - definitions: - theme_properties: + type: object + properties: + colors: type: object + description: Used to override the dashboard colors. Either primary or secondary color must be provided. properties: - colors: - type: object - description: Used to override the dashboard colors. Either primary or secondary color must be provided. - properties: - primary: - type: string - description: Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). - secondary: - type: string - description: Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. - anyOf: - - required: - - primary - - required: - - secondary \ No newline at end of file + primary: + type: string + description: Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). + secondary: + type: string + description: Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. + anyOf: + - required: + - primary + - required: + - secondary \ No newline at end of file diff --git a/runtime/parser/schema/connectors.schema.yaml b/runtime/parser/schema/connectors.schema.yaml index c25b8a0c70f..d90f60af851 100644 --- a/runtime/parser/schema/connectors.schema.yaml +++ b/runtime/parser/schema/connectors.schema.yaml @@ -4,6 +4,42 @@ title: Connector YAML type: object description: | Connector YAML files define how Rill connects to external data sources and OLAP engines. Each connector specifies a driver type and its required connection parameters. + + ## Available Connector Types + + ### _OLAP Engines_ + - [**DuckDB**](#duckdb) - Embedded DuckDB engine (default) + - [**ClickHouse**](#clickhouse) - ClickHouse analytical database + - [**MotherDuck**](#motherduck) - MotherDuck cloud database + - [**Druid**](#druid) - Apache Druid + - [**Pinot**](#pinot) - Apache Pinot + + ### _Data Warehouses_ + - [**Snowflake**](#snowflake) - Snowflake data warehouse + - [**BigQuery**](#bigquery) - Google BigQuery + - [**Redshift**](#redshift) - Amazon Redshift + - [**Athena**](#athena) - Amazon Athena + + ### _Databases_ + - [**PostgreSQL**](#postgres) - PostgreSQL databases + - [**MySQL**](#mysql) - MySQL databases + - [**SQLite**](#sqlite) - SQLite databases + + ### _Cloud Storage_ + - [**GCS**](#gcs) - Google Cloud Storage + - [**S3**](#s3) - Amazon S3 storage + - [**Azure**](#azure) - Azure Blob Storage + + ### _Other_ + - [**HTTPS**](#https) - Public files via HTTP/HTTPS + - [**Salesforce**](#salesforce) - Salesforce data + - [**Slack**](#slack) - Slack data + + :::warning Security Recommendation + For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/build/credentials/) for complete setup instructions. + ::: + + -- allOf: - title: Properties type: object diff --git a/runtime/parser/schema/explore-dashboards.schema.yaml b/runtime/parser/schema/explore-dashboards.schema.yaml index 42a382359f1..ad6bc203752 100644 --- a/runtime/parser/schema/explore-dashboards.schema.yaml +++ b/runtime/parser/schema/explore-dashboards.schema.yaml @@ -14,15 +14,15 @@ allOf: display_name: type: string description: Refers to the display name for the explore dashboard + metrics_view: + type: string + description: Refers to the metrics view resource description: type: string description: Refers to the description of the explore dashboard banner: type: string description: Refers to the custom banner displayed at the header of an explore dashboard - metrics_view: - type: string - description: Refers to the metrics view resource dimensions: description: List of dimension names. Use '*' to select all dimensions (default) $ref: '#/definitions/explore/definitions/field_selector_properties' @@ -31,9 +31,11 @@ allOf: $ref: '#/definitions/explore/definitions/field_selector_properties' theme: oneOf: - - type: string - description: Name of an existing theme to apply to the dashboard - - $ref: '#/definitions/theme/definitions/theme_properties' + - title: Existing theme + type: string + description: Name of an existing theme to apply to the dashboard + - title: Inline theme + type: object description: Inline theme configuration. description: Name of the theme to use. Only one of theme and embedded_theme can be set. time_ranges: @@ -236,14 +238,17 @@ definitions: definitions: field_selector_properties: oneOf: - - type: string + - title: Wildcard(*) selector + type: string const: '*' description: Wildcard(*) selector that includes all available fields in the selection - - type: array + - title: Explicit list of fields + type: array items: type: string description: Explicit list of fields to include in the selection - - type: object + - title: Regex matching + type: object description: 'Advanced matching using regex, DuckDB expression, or exclusion' properties: regex: diff --git a/runtime/parser/schema/metrics_views.schema.yaml b/runtime/parser/schema/metrics_views.schema.yaml index 09eea4243a4..e3e12dfa9a6 100644 --- a/runtime/parser/schema/metrics_views.schema.yaml +++ b/runtime/parser/schema/metrics_views.schema.yaml @@ -7,6 +7,9 @@ allOf: - title: Properties type: object properties: + version: + type: string + description: The version of the metrics view schema type: type: string const: metrics_view @@ -103,7 +106,7 @@ allOf: type: string description: a combination of operators and functions for aggregations window: - description: A measure window can be defined as a keyword string (e.g. 'time' or 'all') or an object with detailed window configuration. + description: A measure window can be defined as a keyword string (e.g. 'time' or 'all') or an object with detailed window configuration. For more information, see the [window functions](/build/metrics-view/advanced-expressions/windows) documentation. anyOf: - type: string enum: @@ -156,11 +159,12 @@ allOf: type: string description: used to configure what value to fill in for missing time buckets. This also works generally as COALESCING over non empty time buckets. minItems: 1 - security: - $ref: '#/definitions/security_policy_properties' - description: Defines a security policy for the dashboard - required: - - type + security: + $ref: '#/definitions/security_policy_properties' + description: Defines a security policy for the dashboard + required: + - type + - version - $ref: '#/definitions/common_properties' definitions: security_policy_properties: diff --git a/runtime/parser/schema/models.schema.yaml b/runtime/parser/schema/models.schema.yaml index 81ebabc7a43..6ac4c160371 100644 --- a/runtime/parser/schema/models.schema.yaml +++ b/runtime/parser/schema/models.schema.yaml @@ -50,5 +50,3 @@ allOf: If unsure, we would generally recommend leaving the defaults and/or reaching out for further guidance! ::: - required: - - type \ No newline at end of file diff --git a/runtime/parser/schema/rillyaml.schema.yaml b/runtime/parser/schema/rillyaml.schema.yaml index 40d88195af2..87dda03d256 100644 --- a/runtime/parser/schema/rillyaml.schema.yaml +++ b/runtime/parser/schema/rillyaml.schema.yaml @@ -165,6 +165,11 @@ allOf: groups: - partners - email: anon@unknown.com + - email: embed@rilldata.com + name: embed + custom_variable_1: Value_1 + custom_variable_2: Value_2 + items: type: object properties: diff --git a/runtime/parser/schema/sources.schema.yaml b/runtime/parser/schema/sources.schema.yaml index e6a6c7a47c3..9995f8601f0 100644 --- a/runtime/parser/schema/sources.schema.yaml +++ b/runtime/parser/schema/sources.schema.yaml @@ -11,8 +11,6 @@ description: | - Migrate to source models via the `type:model` parameter for existing projects - See our [model YAML reference](advanced-models) for current documentation and best practices ::: - - Source YAML files define data sources that can be imported into Rill projects. allOf: - title: Properties type: object @@ -23,7 +21,7 @@ allOf: description: Refers to the resource type and must be `model` connector: type: string - description: Refers to the connector type for the source, see [connectors](/build/connect/) for more information + description: Refers to the connector type for the source, see [connectors](/reference/project-files/connectors) for more information enum: - https - s3 From 77f0ca91cd14907da3e73b665fe47edfd967d646 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Tue, 5 Aug 2025 14:17:51 -0600 Subject: [PATCH 15/32] adding annotatons back --- .../parser/schema/metrics_views.schema.yaml | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/runtime/parser/schema/metrics_views.schema.yaml b/runtime/parser/schema/metrics_views.schema.yaml index e3e12dfa9a6..3141247dcd1 100644 --- a/runtime/parser/schema/metrics_views.schema.yaml +++ b/runtime/parser/schema/metrics_views.schema.yaml @@ -159,6 +159,73 @@ allOf: type: string description: used to configure what value to fill in for missing time buckets. This also works generally as COALESCING over non empty time buckets. minItems: 1 + + annotations: + type: array + description: Used to define annotations that can be displayed on charts + items: + type: object + properties: + name: + type: string + description: A stable identifier for the annotation. Defaults to model or table names when not specified + model: + type: string + description: Refers to the model powering the annotation (either table or model is required). The model must have 'time' and 'description' columns. Optional columns include 'time_end' for range annotations and 'grain' to specify when the annotation should appear based on dashboard grain level. + database: + type: string + description: Refers to the database to use in the OLAP engine (to be used in conjunction with table). Otherwise, will use the default database or schema if not specified + database_schema: + type: string + description: Refers to the schema to use in the OLAP engine (to be used in conjunction with table). Otherwise, will use the default database or schema if not specified + table: + type: string + description: Refers to the table powering the annotation, should be used instead of model for annotations from external OLAP tables (either table or model is required) + connector: + type: string + description: Refers to the connector to use for the annotation + measures: + description: Specifies which measures to apply the annotation to. Applies to all measures if not specified + anyOf: + - type: string + description: Simple field name as a string. + - type: array + description: List of field selectors, each can be a string or an object with detailed configuration. + items: + anyOf: + - type: string + description: Shorthand field selector, interpreted as the name. + - type: object + description: Detailed field selector configuration with name and optional time grain. + properties: + name: + type: string + description: Name of the field to select. + time_grain: + type: string + description: Time grain for time-based dimensions. + enum: + - '' + - ms + - millisecond + - s + - second + - min + - minute + - h + - hour + - d + - day + - w + - week + - month + - q + - quarter + - 'y' + - year + required: + - name + additionalProperties: false security: $ref: '#/definitions/security_policy_properties' description: Defines a security policy for the dashboard From 7c35559ab1120753beaad7805a92e6eeed75871e Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Tue, 5 Aug 2025 14:19:34 -0600 Subject: [PATCH 16/32] gofmt, golint --- cli/cmd/docs/generate_project.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index a69793c04b7..90a110b1467 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -71,17 +71,17 @@ func GenerateProjectDocsCmd(rootCmd *cobra.Command, ch *cmdutil.Helper) *cobra.C requiredMap := getRequiredMapFromNode(resource) resourceFilebuf.WriteString(generateDoc(sidebarPosition, 0, resource, "", requiredMap)) resTitle := getScalarValue(resource, "title") - resId := getScalarValue(resource, "$id") - + resID := getScalarValue(resource, "$id") + // Use $id if available, otherwise fall back to title var fileName string - if resId != "" { + if resID != "" { // Remove .schema.yaml extension and convert to .md - fileName = strings.TrimSuffix(resId, ".schema.yaml") + ".md" + fileName = strings.TrimSuffix(resID, ".schema.yaml") + ".md" } else { fileName = sanitizeFileName(resTitle) + ".md" } - + filePath := filepath.Join(outputDir, fileName) if err := os.WriteFile(filePath, []byte(resourceFilebuf.String()), 0o644); err != nil { return fmt.Errorf("failed writing resource doc: %w", err) @@ -427,13 +427,10 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req // Remove the summary list to avoid duplication with detailed sections if level == 1 { doc.WriteString("\n\n## One of Properties Options") - - } - + } for _, item := range oneOf.Content { if hasType(item) || hasProperties(item) || hasCombinators(item) { - doc.WriteString(generateDoc(sidebarPosition, level, item, indent+" ", getRequiredMapFromNode(item))) } } @@ -453,7 +450,6 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req // AllOf if allOf := getNodeForKey(node, "allOf"); allOf != nil && allOf.Kind == yaml.SequenceNode { - for _, item := range allOf.Content { // Skip oneOf items for connectors since we handle them separately if isConnector && getNodeForKey(item, "oneOf") != nil { From 0ff11e2c56842f094a66cd8b35dd86e2bf349807 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Tue, 5 Aug 2025 16:48:57 -0600 Subject: [PATCH 17/32] fix --- docs/docs/hidden/yaml/connectors.md | 2 +- runtime/parser/schema/connectors.schema.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/hidden/yaml/connectors.md b/docs/docs/hidden/yaml/connectors.md index 5c7af185745..c0aa6d45a00 100644 --- a/docs/docs/hidden/yaml/connectors.md +++ b/docs/docs/hidden/yaml/connectors.md @@ -37,7 +37,7 @@ Connector YAML files define how Rill connects to external data sources and OLAP - [**Slack**](#slack) - Slack data :::warning Security Recommendation -For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/build/credentials/) for complete setup instructions. +For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env.connector..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/build/credentials/) for complete setup instructions. ::: -- diff --git a/runtime/parser/schema/connectors.schema.yaml b/runtime/parser/schema/connectors.schema.yaml index d90f60af851..5d444c78310 100644 --- a/runtime/parser/schema/connectors.schema.yaml +++ b/runtime/parser/schema/connectors.schema.yaml @@ -36,7 +36,7 @@ description: | - [**Slack**](#slack) - Slack data :::warning Security Recommendation - For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/build/credentials/) for complete setup instructions. + For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env.connector..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/build/credentials/) for complete setup instructions. ::: -- From 11b2710eee3a9e7900f734b87f14d8b841412dde Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Wed, 6 Aug 2025 09:03:05 -0600 Subject: [PATCH 18/32] first pass fix, need to review a few other items --- docs/docs/hidden/yaml/advanced-models.md | 6 +- docs/docs/hidden/yaml/apis.md | 18 -- docs/docs/hidden/yaml/connectors.md | 48 ++++- docs/docs/hidden/yaml/metrics_views.md | 186 ------------------ .../parser/schema/advanced-models.schema.yaml | 5 +- runtime/parser/schema/apis.schema.yaml | 22 +-- runtime/parser/schema/connectors.schema.yaml | 22 ++- 7 files changed, 71 insertions(+), 236 deletions(-) delete mode 100644 docs/docs/hidden/yaml/metrics_views.md diff --git a/docs/docs/hidden/yaml/advanced-models.md b/docs/docs/hidden/yaml/advanced-models.md index b909e356351..cfe209b287e 100644 --- a/docs/docs/hidden/yaml/advanced-models.md +++ b/docs/docs/hidden/yaml/advanced-models.md @@ -42,11 +42,7 @@ _[object]_ - Specifies the refresh schedule that Rill should follow to re-ingest ### `connector` -_[string]_ - Refers to the resource type and must be `connector` - -### `driver` - -_[string]_ - The type of connector, see [available connectors](../yaml/connectors#available-connector-types) (required) +_[string]_ - Refers to the resource type and is needed if setting an explicit OLAP engine. IE `clickhouse` ### `sql` diff --git a/docs/docs/hidden/yaml/apis.md b/docs/docs/hidden/yaml/apis.md index 8ad4b3051f8..a0ccaeca2a4 100644 --- a/docs/docs/hidden/yaml/apis.md +++ b/docs/docs/hidden/yaml/apis.md @@ -130,24 +130,6 @@ _[object]_ - Based on resource status _(required)_ - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. -## Common Properties - -### `name` - -_[string]_ - Name is usually inferred from the filename, but can be specified manually. - -### `refs` - -_[array of string]_ - List of resource references - -### `dev` - -_[object]_ - Overrides any properties in development environment. - -### `prod` - -_[object]_ - Overrides any properties in production environment. - ## Examples ```yaml diff --git a/docs/docs/hidden/yaml/connectors.md b/docs/docs/hidden/yaml/connectors.md index c0aa6d45a00..64e0a966f54 100644 --- a/docs/docs/hidden/yaml/connectors.md +++ b/docs/docs/hidden/yaml/connectors.md @@ -40,8 +40,24 @@ Connector YAML files define how Rill connects to external data sources and OLAP For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env.connector..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/build/credentials/) for complete setup instructions. ::: --- +## Common Properties + +### `name` + +_[string]_ - Name is usually inferred from the filename, but can be specified manually. + +### `refs` + +_[array of string]_ - List of resource references + +### `dev` + +_[object]_ - Overrides any properties in development environment. + +### `prod` + +_[object]_ - Overrides any properties in production environment. ## Properties @@ -53,6 +69,36 @@ _[string]_ - Refers to the resource type and must be `connector` _(required)_ _[string]_ - The type of connector, see [available connectors](#available-connector-types) (required) _(required)_ +## Common Properties + +```yaml +type: connector # Must be `connector` (required) +driver: common properties # Must be `common properties` _(required)_ + +name: "example_value" # Name is usually inferred from the filename, but can be specified manually. +refs: "example_value" # List of resource references +dev: "example_value" # Overrides any properties in development environment. +prod: "example_value" # Overrides any properties in production environment. +``` + + + +#### `name` + +_[string]_ - Name is usually inferred from the filename, but can be specified manually. + +#### `refs` + +_[array of string]_ - List of resource references + +#### `dev` + +_[object]_ - Overrides any properties in development environment. + +#### `prod` + +_[object]_ - Overrides any properties in production environment. + ## Athena Configuration properties specific to the athena diff --git a/docs/docs/hidden/yaml/metrics_views.md b/docs/docs/hidden/yaml/metrics_views.md deleted file mode 100644 index dcd196ee12f..00000000000 --- a/docs/docs/hidden/yaml/metrics_views.md +++ /dev/null @@ -1,186 +0,0 @@ ---- -note: GENERATED. DO NOT EDIT. -title: Metrics Views YAML -sidebar_position: 35 ---- - -In your Rill project directory, create a metrics view, `.yaml`, file in the `metrics` directory. Rill will ingest the metric view definition next time you run `rill start`. - -## Properties - -### `type` - -_[string]_ - Refers to the resource type and must be `metrics_view` - -### `display_name` - -_[string]_ - Refers to the display name for the metrics view - -### `description` - -_[string]_ - Refers to the description for the metrics view - -### `ai_instructions` - -_[string]_ - Extra instructions for AI agents. Used to guide natural language question answering and routing. - -### `model` - -_[string]_ - Refers to the model powering the dashboard (either model or table is required) - -### `database` - -_[string]_ - Refers to the database to use in the OLAP engine (to be used in conjunction with table). Otherwise, will use the default database or schema if not specified - -### `database_schema` - -_[string]_ - Refers to the schema to use in the OLAP engine (to be used in conjunction with table). Otherwise, will use the default database or schema if not specified - -### `table` - -_[string]_ - Refers to the table powering the dashboard, should be used instead of model for dashboards create from external OLAP tables (either table or model is required) - -### `timeseries` - -_[string]_ - Refers to the timestamp column from your model that will underlie x-axis data in the line charts. If not specified, the line charts will not appear - -### `watermark` - -_[string]_ - A SQL expression that tells us the max timestamp that the metrics are considered valid for. Usually does not need to be overwritten - -### `smallest_time_grain` - -_[string]_ - Refers to the smallest time granularity the user is allowed to view. The valid values are: millisecond, second, minute, hour, day, week, month, quarter, year - -### `first_day_of_week` - -_[integer]_ - Refers to the first day of the week for time grain aggregation (for example, Sunday instead of Monday). The valid values are 1 through 7 where Monday=1 and Sunday=7 - -### `first_month_of_year` - -_[integer]_ - Refers to the first month of the year for time grain aggregation. The valid values are 1 through 12 where January=1 and December=12 - -### `dimensions` - -_[array of object]_ - Relates to exploring segments or dimensions of your data and filtering the dashboard - - - **`name`** - _[string]_ - a stable identifier for the dimension - - - **`display_name`** - _[string]_ - a display name for your dimension - - - **`description`** - _[string]_ - a freeform text description of the dimension - - - **`column`** - _[string]_ - a categorical column - - - **`expression`** - _[string]_ - a non-aggregate expression such as string_split(domain, '.'). One of column and expression is required but cannot have both at the same time - - - **`unnest`** - _[boolean]_ - if true, allows multi-valued dimension to be unnested (such as lists) and filters will automatically switch to "contains" instead of exact match - - - **`uri`** - _[string, boolean]_ - enable if your dimension is a clickable URL to enable single click navigation (boolean or valid SQL expression) - -### `measures` - -_[array of object]_ - Used to define the numeric aggregates of columns from your data model - - - **`name`** - _[string]_ - a stable identifier for the measure - - - **`display_name`** - _[string]_ - the display name of your measure. - - - **`description`** - _[string]_ - a freeform text description of the dimension - - - **`type`** - _[string]_ - Measure calculation type: "simple" for basic aggregations, "derived" for calculations using other measures, or "time_comparison" for period-over-period analysis. Defaults to "simple" unless dependencies exist. - - - **`expression`** - _[string]_ - a combination of operators and functions for aggregations - - - **`window`** - _[anyOf]_ - A measure window can be defined as a keyword string (e.g. 'time' or 'all') or an object with detailed window configuration. - - - **option 1** - _[string]_ - Shorthand: `time` or `true` means time-partitioned, `all` means non-partitioned. - - - **option 2** - _[object]_ - Detailed window configuration for measure calculations, allowing control over partitioning, ordering, and frame definition. - - - **`partition`** - _[boolean]_ - Controls whether the window is partitioned. When true, calculations are performed within each partition separately. - - - **`order`** - _[anyOf]_ - Specifies the fields to order the window by, determining the sequence of rows within each partition. - - - **option 1** - _[string]_ - Simple field name as a string. - - - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. - - - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. - - - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. - - - **`name`** - _[string]_ - Name of the field to select. _(required)_ - - - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. - - - **`frame`** - _[string]_ - Defines the window frame boundaries for calculations, specifying which rows are included in the window relative to the current row. - - - **`per`** - _[anyOf]_ - for per dimensions - - - **option 1** - _[string]_ - Simple field name as a string. - - - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. - - - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. - - - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. - - - **`name`** - _[string]_ - Name of the field to select. _(required)_ - - - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. - - - **`requires`** - _[anyOf]_ - using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures - - - **option 1** - _[string]_ - Simple field name as a string. - - - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. - - - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. - - - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. - - - **`name`** - _[string]_ - Name of the field to select. _(required)_ - - - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. - - - **`format_preset`** - _[string]_ - Controls the formatting of this measure using a predefined preset. Measures cannot have both `format_preset` and `format_d3`. If neither is supplied, the measure will be formatted using the `humanize` preset by default. - - Available options: - - `humanize`: Round numbers into thousands (K), millions(M), billions (B), etc. - - `none`: Raw output. - - `currency_usd`: Round to 2 decimal points with a dollar sign ($). - - `currency_eur`: Round to 2 decimal points with a euro sign (€). - - `percentage`: Convert a rate into a percentage with a % sign. - - `interval_ms`: Convert milliseconds into human-readable durations like hours (h), days (d), years (y), etc. (optional) - - - - **`format_d3`** - _[string]_ - Controls the formatting of this measure using a [d3-format](https://d3js.org/d3-format) string. If an invalid format string is supplied, the measure will fall back to `format_preset: humanize`. A measure cannot have both `format_preset` and `format_d3`. If neither is provided, the humanize preset is used by default. Example: `format_d3: ".2f"` formats using fixed-point notation with two decimal places. Example: `format_d3: ",.2r"` formats using grouped thousands with two significant digits. (optional) - - - **`format_d3_locale`** - _[object]_ - locale configuration passed through to D3, enabling changing the currency symbol among other things. For details, see the docs for D3's [formatLocale](https://d3js.org/d3-format#formatLocale) - - - **`valid_percent_of_total`** - _[boolean]_ - a boolean indicating whether percent-of-total values should be rendered for this measure - - - **`treat_nulls_as`** - _[string]_ - used to configure what value to fill in for missing time buckets. This also works generally as COALESCING over non empty time buckets. - -### `required` - -_[no type]_ - (no description) - -## Common Properties - -### `name` - -_[string]_ - Name is usually inferred from the filename, but can be specified manually. - -### `refs` - -_[array of string]_ - List of resource references - -### `dev` - -_[object]_ - Overrides any properties in development environment. - -### `prod` - -_[object]_ - Overrides any properties in production environment. \ No newline at end of file diff --git a/runtime/parser/schema/advanced-models.schema.yaml b/runtime/parser/schema/advanced-models.schema.yaml index f9557e53e36..6104e2733f0 100644 --- a/runtime/parser/schema/advanced-models.schema.yaml +++ b/runtime/parser/schema/advanced-models.schema.yaml @@ -34,10 +34,7 @@ allOf: connector: type: string const: connector - description: Refers to the resource type and must be `connector` - driver: - type: string - description: The type of connector, see [available connectors](../yaml/connectors#available-connector-types) (required) + description: Refers to the resource type and is needed if setting an explicit OLAP engine. IE `clickhouse` sql: type: string description: Raw SQL query to run against source diff --git a/runtime/parser/schema/apis.schema.yaml b/runtime/parser/schema/apis.schema.yaml index 7f142813906..5574f07b102 100644 --- a/runtime/parser/schema/apis.schema.yaml +++ b/runtime/parser/schema/apis.schema.yaml @@ -51,7 +51,7 @@ allOf: - $ref: '#/definitions/data_properties' required: - type - - $ref: '#/definitions/common_properties' + definitions: data_properties: oneOf: @@ -218,22 +218,4 @@ definitions: description: SQL expression for row filtering (for row_filter type rules) required: - type - common_properties: - type: object - title: "Common Properties" - properties: - name: - type: string - description: Name is usually inferred from the filename, but can be specified manually. - refs: - type: array - description: 'List of resource references' - items: - type: string - description: A string reference like `` or ``. - dev: - type: object - description: Overrides any properties in development environment. - prod: - type: object - description: Overrides any properties in production environment. \ No newline at end of file + \ No newline at end of file diff --git a/runtime/parser/schema/connectors.schema.yaml b/runtime/parser/schema/connectors.schema.yaml index 5d444c78310..07b30ba42bb 100644 --- a/runtime/parser/schema/connectors.schema.yaml +++ b/runtime/parser/schema/connectors.schema.yaml @@ -39,8 +39,8 @@ description: | For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env.connector..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/build/credentials/) for complete setup instructions. ::: - -- allOf: + - $ref: '#/definitions/common_properties' - title: Properties type: object properties: @@ -75,8 +75,26 @@ allOf: - $ref: '#/definitions/slack' - $ref: '#/definitions/snowflake' - $ref: '#/definitions/sqlite' - definitions: + common_properties: + type: object + title: "Common Properties" + properties: + name: + type: string + description: Name is usually inferred from the filename, but can be specified manually. + refs: + type: array + description: 'List of resource references' + items: + type: string + description: A string reference like `` or ``. + dev: + type: object + description: Overrides any properties in development environment. + prod: + type: object + description: Overrides any properties in production environment. athena: type: object title: Athena From db75a6b2620206e89051fdd70a9d5236ed4858d4 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Wed, 6 Aug 2025 09:15:02 -0600 Subject: [PATCH 19/32] remove toplevel driver --- docs/docs/hidden/yaml/connectors.md | 4 ---- runtime/parser/schema/connectors.schema.yaml | 6 +++--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/docs/docs/hidden/yaml/connectors.md b/docs/docs/hidden/yaml/connectors.md index 64e0a966f54..1fa456adbe2 100644 --- a/docs/docs/hidden/yaml/connectors.md +++ b/docs/docs/hidden/yaml/connectors.md @@ -65,10 +65,6 @@ _[object]_ - Overrides any properties in production environment. _[string]_ - Refers to the resource type and must be `connector` _(required)_ -### `driver` - -_[string]_ - The type of connector, see [available connectors](#available-connector-types) (required) _(required)_ - ## Common Properties ```yaml diff --git a/runtime/parser/schema/connectors.schema.yaml b/runtime/parser/schema/connectors.schema.yaml index 07b30ba42bb..fe4a70e4543 100644 --- a/runtime/parser/schema/connectors.schema.yaml +++ b/runtime/parser/schema/connectors.schema.yaml @@ -48,9 +48,9 @@ allOf: type: string const: connector description: Refers to the resource type and must be `connector` - driver: - type: string - description: The type of connector, see [available connectors](#available-connector-types) (required) + # driver: + # type: string + # description: The type of connector, see [available connectors](#available-connector-types) (required) required: - type - driver From 95ca49a0c772a062f508556d47e676e2ca4921b3 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Wed, 6 Aug 2025 15:39:18 -0600 Subject: [PATCH 20/32] single project file all schemas into one including rillyaml. commented out lines. reworked generate-project.go: - moved out logic for sample in copy, uses schema sample instead - adding in line examples --- cli/cmd/docs/generate_project.go | 321 +-- docs/docs/hidden/yaml/advanced-models.md | 282 +-- docs/docs/hidden/yaml/alerts.md | 120 +- docs/docs/hidden/yaml/apis.md | 70 +- docs/docs/hidden/yaml/canvas-dashboards.md | 152 +- docs/docs/hidden/yaml/connectors.md | 362 +-- docs/docs/hidden/yaml/explore-dashboards.md | 180 +- docs/docs/hidden/yaml/metrics-views.md | 87 +- docs/docs/hidden/yaml/rillyaml.md | 8 + docs/docs/hidden/yaml/sources.md | 29 +- docs/docs/hidden/yaml/themes.md | 49 +- .../advanced-models.schema.yaml | 0 .../parser/{schema => old}/alerts.schema.yaml | 0 .../parser/{schema => old}/apis.schema.yaml | 0 .../canvas-dashboards.schema.yaml | 0 .../{schema => old}/component.schema.yaml | 0 .../{schema => old}/connectors.schema.yaml | 0 .../explore-dashboards.schema.yaml | 0 .../{schema => old}/metrics_views.schema.yaml | 0 .../parser/{schema => old}/models.schema.yaml | 0 .../{schema => old}/rillyaml.schema.yaml | 0 .../{schema => old}/sources.schema.yaml | 0 .../parser/{schema => old}/themes.schema.yaml | 0 runtime/parser/schema/project.schema.yaml | 2205 ++++++++++++++++- 24 files changed, 2725 insertions(+), 1140 deletions(-) rename runtime/parser/{schema => old}/advanced-models.schema.yaml (100%) rename runtime/parser/{schema => old}/alerts.schema.yaml (100%) rename runtime/parser/{schema => old}/apis.schema.yaml (100%) rename runtime/parser/{schema => old}/canvas-dashboards.schema.yaml (100%) rename runtime/parser/{schema => old}/component.schema.yaml (100%) rename runtime/parser/{schema => old}/connectors.schema.yaml (100%) rename runtime/parser/{schema => old}/explore-dashboards.schema.yaml (100%) rename runtime/parser/{schema => old}/metrics_views.schema.yaml (100%) rename runtime/parser/{schema => old}/models.schema.yaml (100%) rename runtime/parser/{schema => old}/rillyaml.schema.yaml (100%) rename runtime/parser/{schema => old}/sources.schema.yaml (100%) rename runtime/parser/{schema => old}/themes.schema.yaml (100%) diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index 90a110b1467..916adbe1b0b 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -29,25 +29,15 @@ func GenerateProjectDocsCmd(rootCmd *cobra.Command, ch *cmdutil.Helper) *cobra.C projectPath := "runtime/parser/schema/project.schema.yaml" projectFilesSchema, err := parseSchemaYAML(projectPath) if err != nil { - return fmt.Errorf("resource schema error: %w", err) + return fmt.Errorf("project schema error: %w", err) } - rillyamlPath := "runtime/parser/schema/rillyaml.schema.yaml" - rillYamlSchema, err := parseSchemaYAML(rillyamlPath) - if err != nil { - return fmt.Errorf("rillyaml schema error: %w", err) - } - - // Add rillyaml to projectFilesSchema's oneOf - oneOfNode := getNodeForKey(projectFilesSchema, "oneOf") - if oneOfNode == nil { - oneOfNode = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} - projectFilesSchema.Content = append(projectFilesSchema.Content, - &yaml.Node{Kind: yaml.ScalarNode, Value: "oneOf"}, - oneOfNode, - ) - } - oneOfNode.Content = append(oneOfNode.Content, rillYamlSchema) + // Load rillyaml schema + // rillyamlPath := "runtime/parser/schema/rillyaml.schema.yaml" + // rillyamlSchema, err := parseSchemaYAML(rillyamlPath) + // if err != nil { + // return fmt.Errorf("rillyaml schema error: %w", err) + // } var projectFilesbuf strings.Builder sidebarPosition := 30 @@ -65,19 +55,26 @@ func GenerateProjectDocsCmd(rootCmd *cobra.Command, ch *cmdutil.Helper) *cobra.C projectFilesbuf.WriteString(fmt.Sprintf("%s\n\n", desc)) projectFilesbuf.WriteString("## Project files types\n\n") + // Get the oneOf node which contains all resource types + oneOfNode := getNodeForKey(projectFilesSchema, "oneOf") + if oneOfNode == nil { + return fmt.Errorf("no oneOf found in project schema") + } + for _, resource := range oneOfNode.Content { sidebarPosition++ var resourceFilebuf strings.Builder requiredMap := getRequiredMapFromNode(resource) - resourceFilebuf.WriteString(generateDoc(sidebarPosition, 0, resource, "", requiredMap)) resTitle := getScalarValue(resource, "title") - resID := getScalarValue(resource, "$id") + resID := getScalarValue(resource, "id") + + resourceFilebuf.WriteString(generateDoc(sidebarPosition, 0, resource, "", requiredMap, projectFilesSchema, resID)) - // Use $id if available, otherwise fall back to title + // Use id if available, otherwise fall back to title var fileName string if resID != "" { - // Remove .schema.yaml extension and convert to .md - fileName = strings.TrimSuffix(resID, ".schema.yaml") + ".md" + // Use the id directly for the filename + fileName = resID + ".md" } else { fileName = sanitizeFileName(resTitle) + ".md" } @@ -89,6 +86,21 @@ func GenerateProjectDocsCmd(rootCmd *cobra.Command, ch *cmdutil.Helper) *cobra.C projectFilesbuf.WriteString(fmt.Sprintf("\n- [%s](%s)", resTitle, fileName)) } + // Generate rillyaml documentation + // sidebarPosition++ + // var rillyamlFilebuf strings.Builder + // rillyamlTitle := getScalarValue(rillyamlSchema, "title") + // // rillyamlDesc := getPrintableDescription(rillyamlSchema, "", "") + // requiredMap := getRequiredMapFromNode(rillyamlSchema) + // rillyamlFilebuf.WriteString(generateDoc(sidebarPosition, 0, rillyamlSchema, "", requiredMap)) + + // rillyamlFileName := "rillyaml.md" + // rillyamlFilePath := filepath.Join(outputDir, rillyamlFileName) + // if err := os.WriteFile(rillyamlFilePath, []byte(rillyamlFilebuf.String()), 0o644); err != nil { + // return fmt.Errorf("failed writing rillyaml doc: %w", err) + // } + // projectFilesbuf.WriteString(fmt.Sprintf("\n- [%s](%s)", rillyamlTitle, rillyamlFileName)) + if err := os.WriteFile(filepath.Join(outputDir, "index.md"), []byte(projectFilesbuf.String()), 0o644); err != nil { return fmt.Errorf("failed writing index.md: %w", err) } @@ -97,6 +109,7 @@ func GenerateProjectDocsCmd(rootCmd *cobra.Command, ch *cmdutil.Helper) *cobra.C return nil }, } + return cmd } @@ -347,7 +360,7 @@ func getPrintableDescription(node *yaml.Node, indentation, defaultValue string) return desc } -func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, requiredFields map[string]bool) string { +func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, requiredFields map[string]bool, rootSchema *yaml.Node, id string) string { if node == nil || node.Kind != yaml.MappingNode { return "" } @@ -356,9 +369,11 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req currentLevel := level title := getScalarValue(node, "title") description := getPrintableDescription(node, indent, "") - - // Check if this is a connector type for special handling - isConnector := title == "Connector YAML" + + // Get the id at level 0, otherwise use the passed id + if level == 0 { + id = getScalarValue(node, "id") + } if level == 0 { doc.WriteString("---\n") @@ -399,14 +414,15 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req propType := getScalarValue(propertiesValueNode, "type") if propType == "object" || propType == "array" || hasCombinators(propertiesValueNode) { newlevel := level + 1 - doc.WriteString(generateDoc(sidebarPosition, newlevel, propertiesValueNode, indent+" ", getRequiredMapFromNode(propertiesValueNode))) + doc.WriteString(generateDoc(sidebarPosition, newlevel, propertiesValueNode, indent+" ", getRequiredMapFromNode(propertiesValueNode), rootSchema, id)) } - if examples := getNodeForKey(propertiesValueNode, "examples"); examples != nil && examples.Kind == yaml.SequenceNode { - for _, example := range examples.Content { - b, err := yaml.Marshal(example) - if err != nil { - panic(err) + if examples := getNodeForKey(propertiesValueNode, "examples"); examples != nil { + if examples.Kind == yaml.SequenceNode { + // Handle array of YAML examples + for _, example := range examples.Content { + b, err := yaml.Marshal(example) + if err != nil { } doc.WriteString(fmt.Sprintf("\n\n```yaml\n%s```", string(b))) } @@ -423,15 +439,60 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req if !(isConnector && level == 1) { if len(oneOf.Content) == 1 { doc.WriteString(generateDoc(sidebarPosition, level, oneOf.Content[0], indent, getRequiredMapFromNode(oneOf.Content[0]))) - } else { - // Remove the summary list to avoid duplication with detailed sections if level == 1 { doc.WriteString("\n\n## One of Properties Options") + + // Generate the connector definition documentation (properties, etc.) + if properties := getNodeForKey(item, "properties"); properties != nil && properties.Kind == yaml.MappingNode { + for j := 0; j < len(properties.Content); j += 2 { + propName := properties.Content[j].Value + propValue := properties.Content[j+1] + required := "" + if requiredFields := getRequiredMapFromNode(item); requiredFields[propName] { + required = "_(required)_" + } + + doc.WriteString(fmt.Sprintf("\n\n#### `%s`\n\n", propName)) + doc.WriteString(fmt.Sprintf("%s - %s %s", + getPrintableType(propValue), + getPrintableDescription(propValue, indent, "(no description)"), + required)) + } } + } + + } else { + // Skip oneOf processing for connectors and data_properties at level 1 since we handle it above + if !(id == "connectors") { + if len(oneOf.Content) == 1 { + doc.WriteString(generateDoc(sidebarPosition, level, oneOf.Content[0], indent, getRequiredMapFromNode(oneOf.Content[0]), rootSchema, id)) + } else { + // Remove the summary list to avoid duplication with detailed sections + if level == 1 { + doc.WriteString("\n\n## One of Properties Options") + } - for _, item := range oneOf.Content { - if hasType(item) || hasProperties(item) || hasCombinators(item) { - doc.WriteString(generateDoc(sidebarPosition, level, item, indent+" ", getRequiredMapFromNode(item))) + for _, item := range oneOf.Content { + if hasType(item) || hasProperties(item) || hasCombinators(item) { + doc.WriteString(generateDoc(sidebarPosition, level, item, indent+" ", getRequiredMapFromNode(item), rootSchema, id)) + } + + // Handle examples for oneOf items + if examples := getNodeForKey(item, "examples"); examples != nil { + if examples.Kind == yaml.SequenceNode { + // Handle array of YAML examples + for _, example := range examples.Content { + b, err := yaml.Marshal(example) + if err != nil { + panic(err) + } + doc.WriteString(fmt.Sprintf("\n\n```yaml\n%s```", string(b))) + } + } else if examples.Kind == yaml.ScalarNode { + // Handle string examples (like markdown code blocks) + doc.WriteString(fmt.Sprintf("\n\n%s", examples.Value)) + } + } } } } @@ -443,7 +504,7 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req for i, item := range anyOf.Content { if hasType(item) || hasProperties(item) || hasCombinators(item) { doc.WriteString(fmt.Sprintf("\n\n%s- **option %d** - %s - %s", indent, i+1, getPrintableType(item), getPrintableDescription(item, indent, "(no description)"))) - doc.WriteString(generateDoc(sidebarPosition, level, item, indent+" ", getRequiredMapFromNode(item))) + doc.WriteString(generateDoc(sidebarPosition, level, item, indent+" ", getRequiredMapFromNode(item), rootSchema, id)) } } } @@ -451,8 +512,9 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req // AllOf if allOf := getNodeForKey(node, "allOf"); allOf != nil && allOf.Kind == yaml.SequenceNode { for _, item := range allOf.Content { - // Skip oneOf items for connectors since we handle them separately - if isConnector && getNodeForKey(item, "oneOf") != nil { + // Special handling for connector oneOf + if id == "connectors" && getNodeForKey(item, "oneOf") != nil { + doc.WriteString(generateDoc(sidebarPosition, level, item, indent, getRequiredMapFromNode(item), rootSchema, id)) continue } @@ -465,15 +527,15 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req doc.WriteString(fmt.Sprintf("\n\n%s**%s**", indent, title)) } thenNode := getNodeForKey(item, "then") - doc.WriteString(generateDoc(sidebarPosition, level, thenNode, indent, getRequiredMapFromNode(item))) + doc.WriteString(generateDoc(sidebarPosition, level, thenNode, indent, getRequiredMapFromNode(item), rootSchema, id)) } else { - doc.WriteString(generateDoc(sidebarPosition, level, item, indent, getRequiredMapFromNode(item))) + doc.WriteString(generateDoc(sidebarPosition, level, item, indent, getRequiredMapFromNode(item), rootSchema, id)) } } } // Definitions (for connectors) - if definitions := getNodeForKey(node, "definitions"); definitions != nil && definitions.Kind == yaml.MappingNode && isConnector && level == 1 { + if definitions := getNodeForKey(node, "definitions"); definitions != nil && definitions.Kind == yaml.MappingNode && id == "connectors" && level == 1 { for i := 0; i < len(definitions.Content); i += 2 { connectorDef := definitions.Content[i+1] @@ -485,8 +547,6 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req if description != "" { doc.WriteString(fmt.Sprintf("%s\n\n", description)) } - // Add YAML example after description - doc.WriteString(generateConnectorExample(title, connectorDef)) } // Generate the connector definition documentation (properties, etc.) but skip the header @@ -499,7 +559,7 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req if requiredFields := getRequiredMapFromNode(connectorDef); requiredFields[propName] { required = "_(required)_" } - + // changed to #### so its not in the sidebar, and more compact doc.WriteString(fmt.Sprintf("\n\n#### `%s`\n\n", propName)) doc.WriteString(fmt.Sprintf("%s - %s %s", getPrintableType(propValue), @@ -511,14 +571,20 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req } // Examples - if examples := getNodeForKey(node, "examples"); examples != nil && examples.Kind == yaml.SequenceNode && currentLevel == 0 { + if examples := getNodeForKey(node, "examples"); examples != nil && currentLevel == 0 { doc.WriteString("\n\n## Examples") - for _, example := range examples.Content { - b, err := yaml.Marshal(example) - if err != nil { - panic(err) + if examples.Kind == yaml.SequenceNode { + // Handle array of YAML examples + for _, example := range examples.Content { + b, err := yaml.Marshal(example) + if err != nil { + panic(err) + } + doc.WriteString(fmt.Sprintf("\n\n```yaml\n%s```", string(b))) } - doc.WriteString(fmt.Sprintf("\n\n```yaml\n%s```", string(b))) + } else if examples.Kind == yaml.ScalarNode { + // Handle string examples (like markdown code blocks) + doc.WriteString(fmt.Sprintf("\n\n%s", examples.Value)) } } @@ -541,154 +607,3 @@ func hasCombinators(node *yaml.Node) bool { return getNodeForKey(node, "anyOf") != nil || getNodeForKey(node, "oneOf") != nil || getNodeForKey(node, "allOf") != nil } -func generateConnectorExample(connectorType string, connectorDef *yaml.Node) string { - if connectorDef == nil { - return "" - } - - var example strings.Builder - example.WriteString("```yaml\n") - example.WriteString("type: connector # Must be `connector` (required)\n") - - // Get the driver from the schema and add it first - driverAdded := false - if driver := getNodeForKey(connectorDef, "driver"); driver != nil { - if constVal := getScalarValue(driver, "const"); constVal != "" { - example.WriteString(fmt.Sprintf("driver: %s # Must be `%s` _(required)_\n\n", constVal, constVal)) - driverAdded = true - } - } - - // Fallback: if driver wasn't found, use the connector type name - if !driverAdded { - // Special case for MotherDuck which uses duckdb driver - if connectorType == "MotherDuck" { - example.WriteString("driver: duckdb # Must be `duckdb` _(required)_\n\n") - } else { - example.WriteString(fmt.Sprintf("driver: %s # Must be `%s` _(required)_\n\n", strings.ToLower(connectorType), strings.ToLower(connectorType))) - } - } - - // Get all properties from the schema - if properties := getNodeForKey(connectorDef, "properties"); properties != nil && properties.Kind == yaml.MappingNode { - for i := 0; i < len(properties.Content); i += 2 { - propName := properties.Content[i].Value - propValue := properties.Content[i+1] - - // Skip the driver property since we already added it - if propName == "driver" { - continue - } - - // Get property description - description := getPrintableDescription(propValue, "", "") - if description == "" { - description = "Property description" - } - - // Get property type and generate appropriate example value - propType := getScalarValue(propValue, "type") - exampleValue := generateExampleValue(propName, propType, propValue) - - // Check if it's required - required := "" - if requiredFields := getRequiredMapFromNode(connectorDef); requiredFields[propName] { - required = " _(required)_" - } - - // Format the line with proper alignment - example.WriteString(fmt.Sprintf("%s: %s", propName, exampleValue)) - - // Add padding for alignment - padding := 35 - len(propName) - len(exampleValue) - if padding > 0 { - example.WriteString(strings.Repeat(" ", padding)) - } - - example.WriteString(fmt.Sprintf("# %s%s\n", description, required)) - } - } - - example.WriteString("```\n\n") - return example.String() -} - -func generateExampleValue(propName, propType string, propNode *yaml.Node) string { - // Check for const values first - if constVal := getScalarValue(propNode, "const"); constVal != "" { - return fmt.Sprintf("%q", constVal) - } - - // Check for enum values - if enum := getNodeForKey(propNode, "enum"); enum != nil && enum.Kind == yaml.SequenceNode && len(enum.Content) > 0 { - return fmt.Sprintf("%q", enum.Content[0].Value) - } - - // Generate based on type and property name - switch propType { - case "string": - // Generate contextual examples based on property name - switch { - case strings.Contains(propName, "key") && strings.Contains(propName, "secret"): - return "\"myawssecretkey\"" - case strings.Contains(propName, "key") && strings.Contains(propName, "access"): - return "\"myawsaccesskey\"" - case strings.Contains(propName, "token"): - return "\"mytemporarytoken\"" - case strings.Contains(propName, "password"): - return "\"mypassword\"" - case strings.Contains(propName, "username"): - return "\"myusername\"" - case strings.Contains(propName, "host"): - return "\"localhost\"" - case strings.Contains(propName, "port"): - return "5432" - case strings.Contains(propName, "database"): - return "\"mydatabase\"" - case strings.Contains(propName, "bucket"): - return "\"my-bucket\"" - case strings.Contains(propName, "region"): - return "\"us-east-1\"" - case strings.Contains(propName, "workgroup"): - return "\"primary\"" - case strings.Contains(propName, "dsn"): - return "\"postgresql://user:pass@localhost:5432/db\"" - case strings.Contains(propName, "path"): - return "\"md:my_db\"" - case strings.Contains(propName, "uri"): - return "\"s3://bucket/path\"" - case strings.Contains(propName, "endpoint"): - return "\"https://api.example.com\"" - case strings.Contains(propName, "account"): - return "\"myaccount\"" - case strings.Contains(propName, "project"): - return "\"myproject\"" - case strings.Contains(propName, "cluster"): - return "\"mycluster\"" - case strings.Contains(propName, "role") && strings.Contains(propName, "arn"): - return "\"arn:aws:iam::123456789012:role/MyRole\"" - case strings.Contains(propName, "session"): - return "\"MySession\"" - case strings.Contains(propName, "external"): - return "\"MyExternalID\"" - case strings.Contains(propName, "location"): - return "\"s3://my-bucket/athena-output/\"" - case strings.Contains(propName, "init_sql"): - return "|\n INSTALL 'motherduck';\n LOAD 'motherduck';\n SET motherduck_token= '{{ .env.motherduck_token }}'" - default: - return "\"example_value\"" - } - case "integer": - return "123" - case "boolean": - // Check if there's a default value - if defaultVal := getScalarValue(propNode, "default"); defaultVal != "" { - return defaultVal - } - return "true" - case "number": - return "123.45" - default: - return "\"example_value\"" - } -} diff --git a/docs/docs/hidden/yaml/advanced-models.md b/docs/docs/hidden/yaml/advanced-models.md index cfe209b287e..253513acaa2 100644 --- a/docs/docs/hidden/yaml/advanced-models.md +++ b/docs/docs/hidden/yaml/advanced-models.md @@ -29,6 +29,12 @@ _[string]_ - Refers to the resource type and must be `model` _(required)_ ### `refresh` _[object]_ - Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying model data +```yaml +refresh: + cron: "* * * * *" + #every: "24h" +``` + - **`cron`** - _[string]_ - A cron expression that defines the execution schedule @@ -64,53 +70,65 @@ _[string]_ - Configure how changes to the model specifications are applied (opti _[oneOf]_ - Refers to the explicitly defined state of your model, cannot be used with partitions (optional) - - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ + - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ + + - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. + + - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. + - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ + - **`args`** - _[object]_ - Arguments to pass to the custom API. - - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ + - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - - **`args`** - _[object]_ - Arguments to pass to the custom API. + - **option 1** - _[string]_ - A simple file path/glob pattern as a string. - - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ + - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. - - **option 1** - _[string]_ - A simple file path/glob pattern as a string. + - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. - - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. + - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. + - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. - - **`resource_status`** - _[object]_ - Based on resource status _(required)_ +```yaml +resource_status: + where_error: true +``` - - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. ### `partitions` _[oneOf]_ - Refers to the how your data is partitioned, cannot be used with state. (optional) - - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ + - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. + - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. - - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ + - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ + - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - - **`args`** - _[object]_ - Arguments to pass to the custom API. + - **`args`** - _[object]_ - Arguments to pass to the custom API. - - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ + - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - - **option 1** - _[string]_ - A simple file path/glob pattern as a string. + - **option 1** - _[string]_ - A simple file path/glob pattern as a string. - - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. + - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. - - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. + - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. - - **`resource_status`** - _[object]_ - Based on resource status _(required)_ + - **`resource_status`** - _[object]_ - Based on resource status _(required)_ + + - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. + +```yaml +resource_status: + where_error: true +``` - - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. ### `materialize` @@ -146,38 +164,6 @@ _[object]_ - to define the properties of output - **`partition_by`** - _[string]_ - Column or expression to partition the table by - **Additional properties for `output` when `connector` is `clickhouse`** - - - **`type`** - _[string]_ - Type to materialize the model into. Can be 'TABLE', 'VIEW' or 'DICTIONARY' - - - **`columns`** - _[string]_ - Column names and types. Can also include indexes. If unspecified, detected from the query. - - - **`engine_full`** - _[string]_ - Full engine definition in SQL format. Can include partition keys, order, TTL, etc. - - - **`engine`** - _[string]_ - Table engine to use. Default is MergeTree - - - **`order_by`** - _[string]_ - ORDER BY clause. - - - **`partition_by`** - _[string]_ - Partition BY clause. - - - **`primary_key`** - _[string]_ - PRIMARY KEY clause. - - - **`sample_by`** - _[string]_ - SAMPLE BY clause. - - - **`ttl`** - _[string]_ - TTL settings for the table or columns. - - - **`table_settings`** - _[string]_ - Table-specific settings. - - - **`query_settings`** - _[string]_ - Settings used in insert/create table as select queries. - - - **`distributed_settings`** - _[string]_ - Settings for distributed table. - - - **`distributed_sharding_key`** - _[string]_ - Sharding key for distributed table. - - - **`dictionary_source_user`** - _[string]_ - User for accessing the source dictionary table (used if type is DICTIONARY). - - - **`dictionary_source_password`** - _[string]_ - Password for the dictionary source user. - ## Common Properties ### `name` @@ -194,190 +180,4 @@ _[object]_ - Overrides any properties in development environment. ### `prod` -_[object]_ - Overrides any properties in production environment. - -## Additional properties when `connector` is [`athena`](./connectors#athena) - -### `output_location` - -_[string]_ - Output location for query results in S3. - -### `workgroup` - -_[string]_ - AWS Athena workgroup to use for queries. - -### `region` - -_[string]_ - AWS region to connect to Athena and the output location. - -## Additional properties when `connector` is [`azure`](./connectors#azure) - -### `path` - -_[string]_ - Path to the source - -### `account` - -_[string]_ - Account identifier - -### `uri` - -_[string]_ - Source URI - -### `extract` - -_[object]_ - Arbitrary key-value pairs for extraction settings - -### `glob` - -_[object]_ - Settings related to glob file matching. - - - **`max_total_size`** - _[integer]_ - Maximum total size (in bytes) matched by glob - - - **`max_objects_matched`** - _[integer]_ - Maximum number of objects matched by glob - - - **`max_objects_listed`** - _[integer]_ - Maximum number of objects listed in glob - - - **`page_size`** - _[integer]_ - Page size for glob listing - -### `batch_size` - -_[string]_ - Size of a batch (e.g., '100MB') - -## Additional properties when `connector` is [`bigquery`](./connectors#bigquery) - -### `project_id` - -_[string]_ - ID of the BigQuery project. - -## Additional properties when `connector` is [`duckdb`](./connectors#duckdb) - -### `path` - -_[string]_ - Path to the data source. - -### `format` - -_[string]_ - Format of the data source (e.g., csv, json, parquet). - -### `pre_exec` - -_[string]_ - refers to SQL queries to run before the main query, available for DuckDB-based models. _(optional)_. Ensure `pre_exec` queries are idempotent. Use `IF NOT EXISTS` statements when applicable. - -### `post_exec` - -_[string]_ - refers to a SQL query that is run after the main query, available for DuckDB-based models. _(optional)_. Ensure `post_exec` queries are idempotent. Use `IF EXISTS` statements when applicable. - -```yaml -pre_exec: ATTACH IF NOT EXISTS 'dbname=postgres host=localhost port=5432 user=postgres password=postgres' AS postgres_db (TYPE POSTGRES); -sql: SELECT * FROM postgres_query('postgres_db', 'SELECT * FROM USERS') -post_exec: DETACH DATABASE IF EXISTS postgres_db -``` - -## Additional properties when `connector` is [`gcs`](./connectors#gcs) - -### `path` - -_[string]_ - Path to the source - -### `uri` - -_[string]_ - Source URI - -### `extract` - -_[object]_ - key-value pairs for extraction settings - -### `glob` - -_[object]_ - Settings related to glob file matching. - - - **`max_total_size`** - _[integer]_ - Maximum total size (in bytes) matched by glob - - - **`max_objects_matched`** - _[integer]_ - Maximum number of objects matched by glob - - - **`max_objects_listed`** - _[integer]_ - Maximum number of objects listed in glob - - - **`page_size`** - _[integer]_ - Page size for glob listing - -### `batch_size` - -_[string]_ - Size of a batch (e.g., '100MB') - -## Additional properties when `connector` is [`redshift`](./connectors#redshift) - -### `output_location` - -_[string]_ - S3 location where query results are stored. - -### `workgroup` - -_[string]_ - Redshift Serverless workgroup to use. - -### `database` - -_[string]_ - Name of the Redshift database. - -### `cluster_identifier` - -_[string]_ - Identifier of the Redshift cluster. - -### `role_arn` - -_[string]_ - ARN of the IAM role to assume for Redshift access. - -### `region` - -_[string]_ - AWS region of the Redshift deployment. - -## Additional properties when `connector` is [`s3`](./connectors#s3) - -### `region` - -_[string]_ - AWS region - -### `endpoint` - -_[string]_ - AWS Endpoint - -### `path` - -_[string]_ - Path to the source - -### `uri` - -_[string]_ - Source URI - -### `extract` - -_[object]_ - key-value pairs for extraction settings - -### `glob` - -_[object]_ - Settings related to glob file matching. - - - **`max_total_size`** - _[integer]_ - Maximum total size (in bytes) matched by glob - - - **`max_objects_matched`** - _[integer]_ - Maximum number of objects matched by glob - - - **`max_objects_listed`** - _[integer]_ - Maximum number of objects listed in glob - - - **`page_size`** - _[integer]_ - Page size for glob listing - -### `batch_size` - -_[string]_ - Size of a batch (e.g., '100MB') - -## Additional properties when `connector` is [`salesforce`](./connectors#salesforce) - -### `soql` - -_[string]_ - SOQL query to execute against the Salesforce instance. - -### `sobject` - -_[string]_ - Salesforce object (e.g., Account, Contact) targeted by the query. - -### `queryAll` - -_[boolean]_ - Whether to include deleted and archived records in the query (uses queryAll API). \ No newline at end of file +_[object]_ - Overrides any properties in production environment. \ No newline at end of file diff --git a/docs/docs/hidden/yaml/alerts.md b/docs/docs/hidden/yaml/alerts.md index 882d0a10a56..79e9cf23ca1 100644 --- a/docs/docs/hidden/yaml/alerts.md +++ b/docs/docs/hidden/yaml/alerts.md @@ -12,13 +12,15 @@ Along with alertings at the dashboard level and can be created via the UI, there _[string]_ - Refers to the resource type and must be `alert` _(required)_ -### `display_name` - -_[string]_ - Refers to the display name for the alert - ### `refresh` -_[object]_ - Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying data _(required)_ +_[object]_ - Refresh schedule for the alert + ```yaml + refresh: + cron: "* * * * *" + #every: "24h" + ``` + - **`cron`** - _[string]_ - A cron expression that defines the execution schedule @@ -30,83 +32,59 @@ _[object]_ - Specifies the refresh schedule that Rill should follow to re-ingest - **`run_in_dev`** - _[boolean]_ - If true, allows the schedule to run in development mode. -### `intervals` - -_[object]_ - define the interval of the alert to check - - - **`duration`** - _[string]_ - a valid ISO8601 duration to define the interval duration - - - **`limit`** - _[integer]_ - maximum number of intervals to check for on invocation - - - **`check_unclosed`** - _[boolean]_ - boolean, whether unclosed intervals should be checked - -### `watermark` +### `display_name` -_[string]_ - Specifies how the watermark is determined for incremental processing. Use 'trigger_time' to set it at runtime or 'inherit' to use the upstream model's watermark. +_[string]_ - Display name for the alert _(required)_ -### `timeout` +### `description` -_[string]_ - define the timeout of the alert in seconds (optional). +_[string]_ - Description for the alert ### `data` -_[oneOf]_ - Specifies one of the options to retrieve or compute the data used by alert _(required)_ - - - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - - - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. - - - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ +_[oneOf]_ - Data source for the alert _(required)_ - - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ + - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - - **`args`** - _[object]_ - Arguments to pass to the custom API. + - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. - - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ + - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - - **option 1** - _[string]_ - A simple file path/glob pattern as a string. + - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. + - **`args`** - _[object]_ - Arguments to pass to the custom API. - - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. + - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - - **`resource_status`** - _[object]_ - Based on resource status _(required)_ + - **option 1** - _[string]_ - A simple file path/glob pattern as a string. - - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. + - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. -### `for` + - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. -_[oneOf]_ - Specifies how user identity or attributes should be evaluated for security policy enforcement. + - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - - **`user_id`** - _[string]_ - The unique user ID used to evaluate security policies. _(required)_ + - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. - - **`user_email`** - _[string]_ - The user's email address used to evaluate security policies. _(required)_ - - - **`attributes`** - _[object]_ - A dictionary of user attributes used to evaluate security policies. _(required)_ - -### `on_recover` - -_[boolean]_ - Send an alert when a previously failing alert recovers. Defaults to false. - -### `on_fail` - -_[boolean]_ - Send an alert when a failure occurs. Defaults to true. +```yaml +resource_status: + where_error: true +``` -### `on_error` -_[boolean]_ - Send an alert when an error occurs during evaluation. Defaults to false. +### `condition` -### `renotify` +_[object]_ - Condition that triggers the alert _(required)_ -_[boolean]_ - Enable repeated notifications for unresolved alerts. Defaults to false. + - **`operator`** - _[string]_ - Comparison operator (gt, lt, eq, etc.) -### `renotify_after` + - **`threshold`** - _[no type]_ - Threshold value for the condition -_[string]_ - Defines the re-notification interval for the alert (e.g., '10m','24h'), equivalent to snooze duration in UI, defaults to 'Off' + - **`measure`** - _[string]_ - Measure to compare against the threshold ### `notify` -_[object]_ - Defines how and where to send notifications. At least one method (email or Slack) is required. _(required)_ +_[object]_ - Notification configuration _(required)_ - **`email`** - _[object]_ - Send notifications via email. @@ -120,10 +98,6 @@ _[object]_ - Defines how and where to send notifications. At least one method (e - **`webhooks`** - _[array of string]_ - An array of Slack webhook URLs to send notifications to. -### `annotations` - -_[object]_ - Key value pair used for annotations - ## Common Properties ### `name` @@ -140,30 +114,4 @@ _[object]_ - Overrides any properties in development environment. ### `prod` -_[object]_ - Overrides any properties in production environment. - -## Examples - -```yaml -# Example: To send alert when data lags by more than 1 day to slack channel #rill-cloud-alerts -type: alert -display_name: Data lags by more than 1 day -# Check the alert every hour. -refresh: - cron: 0 * * * * -# Query that returns non-empty results if the metrics lag by more than 1 day. -data: - sql: |- - SELECT * - FROM - ( - SELECT MAX(event_time) AS max_time - FROM rill_metrics_model - ) - WHERE max_time < NOW() - INTERVAL '1 day' -# Send notifications in Slack. -notify: - slack: - channels: - - '#rill-cloud-alerts' -``` \ No newline at end of file +_[object]_ - Overrides any properties in production environment. \ No newline at end of file diff --git a/docs/docs/hidden/yaml/apis.md b/docs/docs/hidden/yaml/apis.md index a0ccaeca2a4..798f43450ea 100644 --- a/docs/docs/hidden/yaml/apis.md +++ b/docs/docs/hidden/yaml/apis.md @@ -4,7 +4,7 @@ title: API YAML sidebar_position: 39 --- -In your Rill project directory, create a new file name `.yaml` in the `apis` directory containing a custom API definition. See comprehensive documentation on how to define and use [custom APIs](/integrate/custom-apis/index.md) +Custom APIs allow you to create endpoints that can be called to retrieve or manipulate data. ## Properties @@ -75,6 +75,13 @@ _[boolean]_ - Flag to control security inheritance ## SQL Query Executes a raw SQL query against the project's data models. + ```yaml + type: api + + connector: "duckdb" + sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" + ``` + ### `sql` @@ -87,6 +94,12 @@ _[string]_ - specifies the connector to use when running SQL or glob queries. ## Metrics View Query Executes a SQL query that targets a defined metrics view. + ```yaml + type: api + + metrics_sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" + ``` + ### `metrics_sql` @@ -95,6 +108,15 @@ _[string]_ - SQL query that targets a metrics view in the project _(required)_ ## Custom API Call Calls a custom API defined in the project to compute data. + ```yaml + type: api + + api: "user_analytics_api" + args: + start_date: "2024-01-01" + limit: 10 + ``` + ### `api` @@ -107,14 +129,20 @@ _[object]_ - Arguments to pass to the custom API. ## File Glob Query Uses a file-matching pattern (glob) to query data from a connector. + ```yaml + type: api + + glob: "data/*.csv" + ``` + ### `glob` _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - - **option 1** - _[string]_ - A simple file path/glob pattern as a string. + - **option 1** - _[string]_ - A simple file path/glob pattern as a string. - - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. + - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. ### `connector` @@ -128,18 +156,28 @@ Uses the status of a resource as data. _[object]_ - Based on resource status _(required)_ - - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. - -## Examples + - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. ```yaml -# Example: This api returns the top 10 authors by net line changes since the specified date provided in the arguments. -type: api -name: metrics_view_api -metrics_sql: |- - SELECT author_name, net_line_changes - FROM advanced_metrics_view - where author_date > '{{ .args.date }}' - order by net_line_changes DESC - limit 10 -``` \ No newline at end of file +resource_status: + where_error: true +``` + + +## Common Properties + +### `name` + +_[string]_ - Name is usually inferred from the filename, but can be specified manually. + +### `refs` + +_[array of string]_ - List of resource references + +### `dev` + +_[object]_ - Overrides any properties in development environment. + +### `prod` + +_[object]_ - Overrides any properties in production environment. \ No newline at end of file diff --git a/docs/docs/hidden/yaml/canvas-dashboards.md b/docs/docs/hidden/yaml/canvas-dashboards.md index d0a0812bda0..2bb88bd4d0a 100644 --- a/docs/docs/hidden/yaml/canvas-dashboards.md +++ b/docs/docs/hidden/yaml/canvas-dashboards.md @@ -4,7 +4,7 @@ title: Canvas Dashboard YAML sidebar_position: 36 --- -In your Rill project directory, create a canvas dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. +Canvas dashboards provide a flexible way to create custom dashboards with drag-and-drop components. ## Properties @@ -14,7 +14,11 @@ _[string]_ - Refers to the resource type and must be `canvas` _(required)_ ### `display_name` -_[string]_ - Refers to the display name for the canvas +_[string]_ - Refers to the display name for the canvas _(required)_ + +### `description` + +_[string]_ - Description for the canvas dashboard ### `banner` @@ -22,7 +26,7 @@ _[string]_ - Refers to the custom banner displayed at the header of an Canvas da ### `rows` -_[array of object]_ - Refers to all of the rows displayed on the Canvas _(required)_ +_[array of object]_ - Refers to all of the rows displayed on the Canvas - **`height`** - _[string]_ - Height of the row in px @@ -46,130 +50,102 @@ _[array of object]_ - Refers to all of the rows displayed on the Canvas _(requir - **`width`** - _[string, integer]_ - Width of the component (can be a number or string with unit) -### `max_width` - -_[integer]_ - Max width in pixels of the canvas - -### `gap_x` - -_[integer]_ - Horizontal gap in pixels of the canvas - -### `gap_y` - -_[integer]_ - Vertical gap in pixels of the canvas - -### `theme` - -_[oneOf]_ - Theme configuration. Can be either a string reference to an existing theme or an inline theme configuration object. - - - **`colors`** - _[object]_ - Used to override the dashboard colors. Either primary or secondary color must be provided. - - - **`primary`** - _[string]_ - Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). - - - **`secondary`** - _[string]_ - Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. - -### `allow_custom_time_range` - -_[boolean]_ - Defaults to true, when set to false it will hide the ability to set a custom time range for the user. - -### `time_ranges` - -_[array of oneOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' - - - **`range`** - _[string]_ - A valid ISO 8601 duration or one of the Rill ISO 8601 extensions for the selection _(required)_ +### `components` - - **`comparison_offsets`** - _[array of oneOf]_ - List of time comparison options for this time range selection (optional). Must be one of the Rill ISO 8601 extensions +_[array of object]_ - Array of components to display on the canvas - - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) + - **`type`** - _[string]_ - Type of component (chart, table, text, etc.) - - **`range`** - _[string]_ - Custom time range for comparison period + - **`title`** - _[string]_ - Title for the component -### `time_zones` - -_[array of string]_ - Refers to the time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) - -### `filters` - -_[object]_ - Indicates if filters should be enabled for the canvas. + - **`data`** - _[oneOf]_ - Data source for the component - - **`enable`** - _[boolean]_ - Toggles filtering functionality for the canvas dashboard. + - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ -### `defaults` + - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. -_[object]_ - Preset UI state to show by default + - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - - **`time_range`** - _[string]_ - Default time range to display when the dashboard loads + - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - - **`comparison_mode`** - _[string]_ - Default comparison mode for metrics (none, time, or dimension) + - **`args`** - _[object]_ - Arguments to pass to the custom API. - - **`comparison_dimension`** - _[string]_ - Default dimension to use for comparison when comparison_mode is 'dimension' + - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ -### `variables` + - **option 1** - _[string]_ - A simple file path/glob pattern as a string. -_[array of object]_ - Variables that can be used in the canvas + - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. - - **`name`** - _[string]_ - Unique identifier for the variable _(required)_ + - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. - - **`type`** - _[string]_ - Data type of the variable (e.g., string, number, boolean) _(required)_ + - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - - **`value`** - _[string, number, boolean, object, array]_ - Default value for the variable. Can be any valid JSON value type + - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. -### `security` - -_[object]_ - Defines security rules and access control policies for resources - - - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. - - - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause +```yaml +resource_status: + where_error: true +``` - - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded - - **`if`** - _[string]_ - Expression to decide if the column should be included or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ + - **`layout`** - _[object]_ - Layout configuration for the component - - **`names`** - _[anyOf]_ - List of fields to include. Should match the name of one of the dashboard's dimensions or measures _(required)_ + - **`style`** - _[object]_ - Styling configuration for the component - - **option 1** - _[array of string]_ - List of specific field names to include +### `max_width` - - **option 2** - _[string]_ - Wildcard '*' to include all fields +_[integer]_ - Max width in pixels of the canvas - - **`exclude`** - _[array of object]_ - List of dimension or measure names to exclude from the dashboard. If exclude is defined all other dimensions and measures are included +### `gap_x` - - **`if`** - _[string]_ - Expression to decide if the column should be excluded or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ +_[integer]_ - Horizontal gap in pixels of the canvas - - **`names`** - _[anyOf]_ - List of fields to exclude. Should match the name of one of the dashboard's dimensions or measures _(required)_ +### `gap_y` - - **option 1** - _[array of string]_ - List of specific field names to exclude +_[integer]_ - Vertical gap in pixels of the canvas - - **option 2** - _[string]_ - Wildcard '*' to exclude all fields +### `theme` - - **`rules`** - _[array of object]_ - List of detailed security rules that can be used to define complex access control policies +_[oneOf]_ - Theme configuration. Can be either a string reference to an existing theme or an inline theme configuration object. - - **`type`** - _[string]_ - Type of security rule - access (overall access), field_access (field-level access), or row_filter (row-level filtering) _(required)_ +### `allow_custom_time_range` - - **`action`** - _[string]_ - Whether to allow or deny access for this rule +_[boolean]_ - Defaults to true, when set to false it will hide the ability to set a custom time range for the user. - - **`if`** - _[string]_ - Conditional expression that determines when this rule applies. Must be a valid SQL expression that evaluates to a boolean +### `time_ranges` - - **`names`** - _[array of string]_ - List of field names this rule applies to (for field_access type rules) +_[array of oneOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' + ```yaml + time_ranges: + - PT15M // Simplified syntax to specify only the range + - PT1H + - PT6H + - P7D + - range: P5D // Advanced syntax to specify comparison_offsets as well + - P4W + - rill-TD // Today + - rill-WTD // Week-To-date + ``` + - - **`all`** - _[boolean]_ - When true, applies the rule to all fields (for field_access type rules) + - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ - - **`sql`** - _[string]_ - SQL expression for row filtering (for row_filter type rules) + - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) -## Common Properties + - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) -### `name` + - **`range`** - _[string]_ - Custom time range for comparison period -_[string]_ - Name is usually inferred from the filename, but can be specified manually. +### `time_zones` -### `refs` +_[array of string]_ - Refers to the time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) -_[array of string]_ - List of resource references +### `filters` -### `dev` +_[object]_ - Indicates if filters should be enabled for the canvas. -_[object]_ - Overrides any properties in development environment. +### `security` -### `prod` +_[object]_ - Defines security rules and access control policies for dashboards (without row filtering) -_[object]_ - Overrides any properties in production environment. \ No newline at end of file + - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. \ No newline at end of file diff --git a/docs/docs/hidden/yaml/connectors.md b/docs/docs/hidden/yaml/connectors.md index 1fa456adbe2..204da86883e 100644 --- a/docs/docs/hidden/yaml/connectors.md +++ b/docs/docs/hidden/yaml/connectors.md @@ -41,24 +41,6 @@ For all credential parameters (passwords, tokens, keys), use environment variabl ::: -## Common Properties - -### `name` - -_[string]_ - Name is usually inferred from the filename, but can be specified manually. - -### `refs` - -_[array of string]_ - List of resource references - -### `dev` - -_[object]_ - Overrides any properties in development environment. - -### `prod` - -_[object]_ - Overrides any properties in production environment. - ## Properties ### `type` @@ -67,53 +49,27 @@ _[string]_ - Refers to the resource type and must be `connector` _(required)_ ## Common Properties -```yaml -type: connector # Must be `connector` (required) -driver: common properties # Must be `common properties` _(required)_ - -name: "example_value" # Name is usually inferred from the filename, but can be specified manually. -refs: "example_value" # List of resource references -dev: "example_value" # Overrides any properties in development environment. -prod: "example_value" # Overrides any properties in production environment. -``` - - - -#### `name` +### `name` _[string]_ - Name is usually inferred from the filename, but can be specified manually. -#### `refs` +### `refs` _[array of string]_ - List of resource references -#### `dev` +### `dev` _[object]_ - Overrides any properties in development environment. -#### `prod` +### `prod` _[object]_ - Overrides any properties in production environment. -## Athena +## Available Connector Types -Configuration properties specific to the athena -```yaml -type: connector # Must be `connector` (required) -driver: athena # Must be `athena` _(required)_ -aws_access_key_id: "myawsaccesskey" # AWS Access Key ID used for authentication. Required when using static credentials directly or as base credentials for assuming a role. -aws_secret_access_key: "myawssecretkey"# AWS Secret Access Key paired with the Access Key ID. Required when using static credentials directly or as base credentials for assuming a role. -aws_access_token: "mytemporarytoken" # AWS session token used with temporary credentials. Required only if the Access Key and Secret Key are part of a temporary session credentials. -role_arn: "arn:aws:iam::123456789012:role/MyRole"# ARN of the IAM role to assume. When specified, the SDK uses the base credentials to call STS AssumeRole and obtain temporary credentials scoped to this role. -role_session_name: "MySession" # Session name to associate with the STS AssumeRole session. Used only if 'role_arn' is specified. Useful for identifying and auditing the session. -external_id: "MyExternalID" # External ID required by some roles when assuming them, typically for cross-account access. Used only if 'role_arn' is specified and the role's trust policy requires it. -workgroup: "primary" # Athena workgroup to use for query execution. Defaults to 'primary' if not specified. -output_location: "s3://my-bucket/athena-output/"# S3 URI where Athena query results should be stored (e.g., s3://your-bucket/athena/results/). Optional if the selected workgroup has a default result configuration. -aws_region: "us-east-1" # AWS region where Athena and the result S3 bucket are located (e.g., us-east-1). Defaults to 'us-east-1' if not specified. -allow_host_access: true # Allow the Athena client to access host environment configurations such as environment variables or local AWS credential files. Defaults to true, enabling use of credentials and settings from the host environment unless explicitly disabled. -``` +### Athena @@ -161,21 +117,7 @@ _[string]_ - AWS region where Athena and the result S3 bucket are located (e.g., _[boolean]_ - Allow the Athena client to access host environment configurations such as environment variables or local AWS credential files. Defaults to true, enabling use of credentials and settings from the host environment unless explicitly disabled. -## Azure - -Configuration properties specific to the azure - -```yaml -type: connector # Must be `connector` (required) -driver: azure # Must be `azure` _(required)_ - -azure_storage_account: "myaccount" # Azure storage account name -azure_storage_key: "example_value" # Azure storage access key -azure_storage_sas_token: "mytemporarytoken"# Optional azure SAS token for authentication -azure_storage_connection_string: "example_value"# Optional azure connection string for storage account -azure_storage_bucket: "my-bucket" # Name of the Azure Blob Storage container (equivalent to an S3 bucket) _(required)_ -allow_host_access: true # Allow access to host environment configuration -``` +### Azure @@ -191,6 +133,10 @@ _[string]_ - Azure storage account name _[string]_ - Azure storage access key +#### `azure_storage_bucket` + +_[string]_ - Name of the Azure Blob Storage container (equivalent to an S3 bucket) _(required)_ + #### `azure_storage_sas_token` _[string]_ - Optional azure SAS token for authentication @@ -199,26 +145,11 @@ _[string]_ - Optional azure SAS token for authentication _[string]_ - Optional azure connection string for storage account -#### `azure_storage_bucket` - -_[string]_ - Name of the Azure Blob Storage container (equivalent to an S3 bucket) _(required)_ - #### `allow_host_access` -_[boolean]_ - Allow access to host environment configuration - -## BigQuery +_[boolean]_ - Allow access to host environment configuratio -Configuration properties specific to the bigquery - -```yaml -type: connector # Must be `connector` (required) -driver: bigquery # Must be `bigquery` _(required)_ - -google_application_credentials: "example_value"# Raw contents of the Google Cloud service account key (in JSON format) used for authentication. -project_id: "myproject" # ID of the Google Cloud project to use for BigQuery operations. This can be omitted only if the project ID is included in the service account key. -allow_host_access: true # Enable the BigQuery client to use credentials from the host environment when no service account JSON is provided. This includes Application Default Credentials from environment variables, local credential files, or the Google Compute Engine metadata server. Defaults to true, allowing seamless authentication in GCP environments. -``` +### BigQuery @@ -232,40 +163,21 @@ _[string]_ - Raw contents of the Google Cloud service account key (in JSON forma #### `project_id` -_[string]_ - ID of the Google Cloud project to use for BigQuery operations. This can be omitted only if the project ID is included in the service account key. +_[string]_ - Google Cloud project ID + +#### `dataset_id` + +_[string]_ - BigQuery dataset ID + +#### `location` + +_[string]_ - BigQuery dataset location #### `allow_host_access` _[boolean]_ - Enable the BigQuery client to use credentials from the host environment when no service account JSON is provided. This includes Application Default Credentials from environment variables, local credential files, or the Google Compute Engine metadata server. Defaults to true, allowing seamless authentication in GCP environments. -## ClickHouse - -Configuration properties specific to the clickhouse - -```yaml -type: connector # Must be `connector` (required) -driver: clickhouse # Must be `clickhouse` _(required)_ - -managed: true # `true` means Rill will provision the connector using the default provisioner. `false` disables automatic provisioning. -mode: "example_value" # `read` - Controls the operation mode for the ClickHouse connection. Defaults to 'read' for safe operation with external databases. Set to 'readwrite' to enable model creation and table mutations. Note: When 'managed: true', this is automatically set to 'readwrite'. -dsn: "postgresql://user:pass@localhost:5432/db"# DSN(Data Source Name) for the ClickHouse connection -username: "myusername" # Username for authentication -password: "mypassword" # Password for authentication -host: "localhost" # Host where the ClickHouse instance is running -port: 123 # Port where the ClickHouse instance is accessible -database: "mydatabase" # Name of the ClickHouse database within the cluster -ssl: true # Indicates whether a secured SSL connection is required -cluster: "mycluster" # Cluster name, required for running distributed queries -log_queries: true # Controls whether to log raw SQL queries -settings_override: "example_value" # override the default settings used in queries. example `readonly = 1, session_timezone = 'UTC'` -embed_port: 123 # Port to run ClickHouse locally (0 for random port) -can_scale_to_zero: true # Indicates if the database can scale to zero -max_open_conns: 123 # Maximum number of open connections to the database -max_idle_conns: 123 # Maximum number of idle connections in the pool -dial_timeout: "example_value" # Timeout for dialing the ClickHouse server -conn_max_lifetime: "example_value" # Maximum time a connection may be reused -read_timeout: "example_value" # Maximum time for a connection to read data -``` +### ClickHouse @@ -349,24 +261,7 @@ _[string]_ - Maximum time a connection may be reused _[string]_ - Maximum time for a connection to read data -## Druid - -Configuration properties specific to the druid - -```yaml -type: connector # Must be `connector` (required) -driver: druid # Must be `druid` _(required)_ - -dsn: "postgresql://user:pass@localhost:5432/db"# Data Source Name (DSN) for connecting to Druid _(required)_ -username: "myusername" # Username for authenticating with Druid -password: "mypassword" # Password for authenticating with Druid -host: "localhost" # Hostname of the Druid coordinator or broker -port: 123 # Port number of the Druid service -ssl: true # Enable SSL for secure connection -log_queries: true # Log raw SQL queries sent to Druid -max_open_conns: 123 # Maximum number of open database connections (0 = default, -1 = unlimited) -skip_version_check: true # Skip checking Druid version compatibility -``` +### Druid @@ -410,30 +305,7 @@ _[integer]_ - Maximum number of open database connections (0 = default, -1 = unl _[boolean]_ - Skip checking Druid version compatibility -## DuckDB - -Configuration properties specific to the duckdb - -```yaml -type: connector # Must be `connector` (required) -driver: duckdb # Must be `duckdb` _(required)_ - -pool_size: 123 # Number of concurrent connections and queries allowed -allow_host_access: true # Whether access to the local environment and file system is allowed -cpu: 123 # Number of CPU cores available to the database -memory_limit_gb: 123 # Amount of memory in GB available to the database -read_write_ratio: 123.45 # Ratio of resources allocated to the read database; used to divide CPU and memory -init_sql: | - INSTALL 'motherduck'; - LOAD 'motherduck'; - SET motherduck_token= '{{ .env.motherduck_token }}'# is executed during database initialization. -conn_init_sql: | - INSTALL 'motherduck'; - LOAD 'motherduck'; - SET motherduck_token= '{{ .env.motherduck_token }}'# is executed when a new connection is initialized. -secrets: "example_value" # Comma-separated list of other connector names to create temporary secrets for in DuckDB before executing a model. -log_queries: true # Whether to log raw SQL queries executed through OLAP -``` +### DuckDB @@ -465,10 +337,6 @@ _[number]_ - Ratio of resources allocated to the read database; used to divide C _[string]_ - is executed during database initialization. -#### `conn_init_sql` - -_[string]_ - is executed when a new connection is initialized. - #### `secrets` _[string]_ - Comma-separated list of other connector names to create temporary secrets for in DuckDB before executing a model. @@ -477,20 +345,7 @@ _[string]_ - Comma-separated list of other connector names to create temporary s _[boolean]_ - Whether to log raw SQL queries executed through OLAP -## GCS - -Configuration properties specific to the gcs - -```yaml -type: connector # Must be `connector` (required) -driver: gcs # Must be `gcs` _(required)_ - -google_application_credentials: "example_value"# Google Cloud credentials JSON string -bucket: "my-bucket" # Name of gcs bucket _(required)_ -allow_host_access: true # Allow access to host environment configuration -key_id: "example_value" # Optional S3-compatible Key ID when used in compatibility mode -secret: "example_value" # Optional S3-compatible Secret when used in compatibility mode -``` +### GCS @@ -518,17 +373,7 @@ _[string]_ - Optional S3-compatible Key ID when used in compatibility mode _[string]_ - Optional S3-compatible Secret when used in compatibility mode -## HTTPS - -Configuration properties specific to the https - -```yaml -type: connector # Must be `connector` (required) -driver: https # Must be `https` _(required)_ - -path: "md:my_db" # The full HTTPS URI to fetch data from _(required)_ -headers: "example_value" # HTTP headers to include in the request -``` +### HTTPS @@ -544,20 +389,7 @@ _[string]_ - The full HTTPS URI to fetch data from _(required)_ _[object]_ - HTTP headers to include in the request -## MotherDuck - -Configuration properties specific to the motherduck - -```yaml -type: connector # Must be `connector` (required) -driver: duckdb # Must be `duckdb` _(required)_ - -path: "md:my_db" # Path to your MD database _(required)_ -init_sql: | - INSTALL 'motherduck'; - LOAD 'motherduck'; - SET motherduck_token= '{{ .env.motherduck_token }}'# SQL executed during database initialization. _(required)_ -``` +### MotherDuck @@ -573,22 +405,7 @@ _[string]_ - Path to your MD database _(required)_ _[string]_ - SQL executed during database initialization. _(required)_ -## MySQL - -Configuration properties specific to the mysql - -```yaml -type: connector # Must be `connector` (required) -driver: mysql # Must be `mysql` _(required)_ - -dsn: "postgresql://user:pass@localhost:5432/db"# DSN(Data Source Name) for the mysql connection -host: "localhost" # Hostname of the MySQL server -port: 123 # Port number for the MySQL server -database: "mydatabase" # Name of the MySQL database -user: "example_value" # Username for authentication -password: "mypassword" # Password for authentication -ssl_mode: "example_value" # SSL mode can be DISABLED, PREFERRED or REQUIRED -``` +### MySQL @@ -624,25 +441,7 @@ _[string]_ - Password for authentication _[string]_ - SSL mode can be DISABLED, PREFERRED or REQUIRED -## Pinot - -Configuration properties specific to the pinot - -```yaml -type: connector # Must be `connector` (required) -driver: pinot # Must be `pinot` _(required)_ - -dsn: "postgresql://user:pass@localhost:5432/db"# DSN(Data Source Name) for the Pinot connection _(required)_ -username: "myusername" # Username for authenticating with Pinot -password: "mypassword" # Password for authenticating with Pinot -broker_host: "localhost" # Hostname of the Pinot broker _(required)_ -broker_port: 123 # Port number for the Pinot broker -controller_host: "localhost" # Hostname of the Pinot controller _(required)_ -controller_port: 123 # Port number for the Pinot controller -ssl: true # Enable SSL connection to Pinot -log_queries: true # Log raw SQL queries executed through Pinot -max_open_conns: 123 # Maximum number of open connections to the Pinot database -``` +### Pinot @@ -690,22 +489,7 @@ _[boolean]_ - Log raw SQL queries executed through Pinot _[integer]_ - Maximum number of open connections to the Pinot database -## Postgres - -Configuration properties specific to the postgres - -```yaml -type: connector # Must be `connector` (required) -driver: postgres # Must be `postgres` _(required)_ - -dsn: "postgresql://user:pass@localhost:5432/db"# DSN(Data Source Name) for the postgres connection -host: "localhost" # Hostname of the Postgres server -port: 5432 # Port number for the Postgres server -dbname: "example_value" # Name of the Postgres database -user: "example_value" # Username for authentication -password: "mypassword" # Password for authentication -sslmode: "example_value" # SSL mode can be disable, allow, prefer or require -``` +### Postgres @@ -741,22 +525,7 @@ _[string]_ - Password for authentication _[string]_ - SSL mode can be disable, allow, prefer or require -## Redshift - -Configuration properties specific to the redshift - -```yaml -type: connector # Must be `connector` (required) -driver: redshift # Must be `redshift` _(required)_ - -aws_access_key_id: "myawsaccesskey" # AWS Access Key ID used for authenticating with Redshift. _(required)_ -aws_secret_access_key: "myawssecretkey"# AWS Secret Access Key used for authenticating with Redshift. _(required)_ -aws_access_token: "mytemporarytoken" # AWS Session Token for temporary credentials (optional). -region: "us-east-1" # AWS region where the Redshift cluster or workgroup is hosted (e.g., 'us-east-1'). -database: "mydatabase" # Name of the Redshift database to query. _(required)_ -workgroup: "primary" # Workgroup name for Redshift Serverless, in case of provisioned Redshift clusters use 'cluster_identifier'. -cluster_identifier: "mycluster" # Cluster identifier for provisioned Redshift clusters, in case of Redshift Serverless use 'workgroup' . -``` +### Redshift @@ -792,23 +561,7 @@ _[string]_ - Workgroup name for Redshift Serverless, in case of provisioned Reds _[string]_ - Cluster identifier for provisioned Redshift clusters, in case of Redshift Serverless use 'workgroup' . -## S3 - -Configuration properties specific to the s3 - -```yaml -type: connector # Must be `connector` (required) -driver: s3 # Must be `s3` _(required)_ - -aws_access_key_id: "myawsaccesskey" # AWS Access Key ID used for authentication -aws_secret_access_key: "myawssecretkey"# AWS Secret Access Key used for authentication -aws_access_token: "mytemporarytoken" # Optional AWS session token for temporary credentials -bucket: "my-bucket" # Name of s3 bucket _(required)_ -endpoint: "https://api.example.com" # Optional custom endpoint URL for S3-compatible storage -region: "us-east-1" # AWS region of the S3 bucket -allow_host_access: true # Allow access to host environment configuration -retain_files: true # Whether to retain intermediate files after processing -``` +### S3 @@ -848,20 +601,7 @@ _[boolean]_ - Allow access to host environment configuration _[boolean]_ - Whether to retain intermediate files after processing -## Salesforce - -Configuration properties specific to the salesforce - -```yaml -type: connector # Must be `connector` (required) -driver: salesforce # Must be `salesforce` _(required)_ - -username: "myusername" # Salesforce account username _(required)_ -password: "mypassword" # Salesforce account password (secret) -key: "example_value" # Authentication key for Salesforce (secret) -endpoint: "https://api.example.com" # Salesforce API endpoint URL _(required)_ -client_id: "example_value" # Client ID used for Salesforce OAuth authentication -``` +### Salesforce @@ -889,16 +629,7 @@ _[string]_ - Salesforce API endpoint URL _(required)_ _[string]_ - Client ID used for Salesforce OAuth authentication -## Slack - -Configuration properties specific to the slack - -```yaml -type: connector # Must be `connector` (required) -driver: slack # Must be `slack` _(required)_ - -bot_token: "mytemporarytoken" # Bot token used for authenticating Slack API requests _(required)_ -``` +### Slack @@ -910,17 +641,7 @@ _[string]_ - Refers to the driver type and must be driver `slack` _(required)_ _[string]_ - Bot token used for authenticating Slack API requests _(required)_ -## Snowflake - -Configuration properties specific to the snowflake - -```yaml -type: connector # Must be `connector` (required) -driver: snowflake # Must be `snowflake` _(required)_ - -dsn: "postgresql://user:pass@localhost:5432/db"# DSN (Data Source Name) for the Snowflake connection _(required)_ -parallel_fetch_limit: 123 # Maximum number of concurrent fetches during query execution -``` +### Snowflake @@ -936,16 +657,7 @@ _[string]_ - DSN (Data Source Name) for the Snowflake connection _(required)_ _[integer]_ - Maximum number of concurrent fetches during query execution -## SQLite - -Configuration properties specific to the sqlite - -```yaml -type: connector # Must be `connector` (required) -driver: sqlite # Must be `sqlite` _(required)_ - -dsn: "postgresql://user:pass@localhost:5432/db"# DSN(Data Source Name) for the sqlite connection _(required)_ -``` +### SQLite diff --git a/docs/docs/hidden/yaml/explore-dashboards.md b/docs/docs/hidden/yaml/explore-dashboards.md index 69cb970a301..eaf82323f0d 100644 --- a/docs/docs/hidden/yaml/explore-dashboards.md +++ b/docs/docs/hidden/yaml/explore-dashboards.md @@ -4,7 +4,7 @@ title: Explore Dashboard YAML sidebar_position: 37 --- -In your Rill project directory, create a explore dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. +Explore dashboards provide an interactive way to explore data with predefined metrics and dimensions. ## Properties @@ -14,11 +14,11 @@ _[string]_ - Refers to the resource type and must be `explore` _(required)_ ### `display_name` -_[string]_ - Refers to the display name for the explore dashboard +_[string]_ - Refers to the display name for the explore dashboard _(required)_ ### `metrics_view` -_[string]_ - Refers to the metrics view resource +_[string]_ - Refers to the metrics view resource _(required)_ ### `description` @@ -30,140 +30,172 @@ _[string]_ - Refers to the custom banner displayed at the header of an explore d ### `dimensions` -_[oneOf]_ - List of dimension names. Use '*' to select all dimensions (default) +_[anyOf]_ - List of dimension names. Use '*' to select all dimensions (default) + ```yaml + # Example: Select a dimension + dimensions: + - country - - **`regex`** - _[string]_ - Select dimensions using a regular expression + # Example: Select all dimensions except one + dimensions: + exclude: + - country - - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic + # Example: Select all dimensions that match a regex + dimensions: + regex: "^public_.*$" + ``` + - - **`exclude`** - _[object]_ - Select all dimensions except those listed here + - **option 1** - _[string]_ - Simple field name as a string. -### `measures` - -_[oneOf]_ - List of measure names. Use ''*'' to select all measures (default) + - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. - - **`regex`** - _[string]_ - Select dimensions using a regular expression + - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. - - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic + - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. - - **`exclude`** - _[object]_ - Select all dimensions except those listed here + - **`name`** - _[string]_ - Name of the field to select. _(required)_ -### `theme` - -_[oneOf]_ - Name of the theme to use. Only one of theme and embedded_theme can be set. + - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. -### `time_ranges` +### `measures` -_[array of oneOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' +_[anyOf]_ - List of measure names. Use '*' to select all measures (default) - - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ + - **option 1** - _[string]_ - Simple field name as a string. - - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. - - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) + - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. - - **`range`** - _[string]_ - Custom time range for comparison period + - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. -### `time_zones` + - **`name`** - _[string]_ - Name of the field to select. _(required)_ -_[array of string]_ - Refers to the time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) + - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. -### `lock_time_zone` +### `theme` -_[boolean]_ - When true, the dashboard will be locked to the first time provided in the time_zones list. When no time_zones are provided, the dashboard will be locked to UTC +_[oneOf]_ - Name of the theme to use. Only one of theme and embedded_theme can be set. -### `allow_custom_time_range` +### `time_range` -_[boolean]_ - Defaults to true, when set to false it will hide the ability to set a custom time range for the user. +_[oneOf]_ - Default time range for the dashboard -### `defaults` + - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ -_[object]_ - defines the defaults YAML struct + - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) - - **`dimensions`** - _[oneOf]_ - Provides the default dimensions to load on viewing the dashboard + - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) - - **`regex`** - _[string]_ - Select dimensions using a regular expression + - **`range`** - _[string]_ - Custom time range for comparison period - - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic +### `time_ranges` - - **`exclude`** - _[object]_ - Select all dimensions except those listed here +_[array of oneOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' + ```yaml + time_ranges: + - PT15M // Simplified syntax to specify only the range + - PT1H + - PT6H + - P7D + - range: P5D // Advanced syntax to specify comparison_offsets as well + - P4W + - rill-TD // Today + - rill-WTD // Week-To-date + ``` + - - **`measures`** - _[oneOf]_ - Provides the default measures to load on viewing the dashboard + - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ - - **`regex`** - _[string]_ - Select dimensions using a regular expression + - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) - - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic + - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) - - **`exclude`** - _[object]_ - Select all dimensions except those listed here + - **`range`** - _[string]_ - Custom time range for comparison period - - **`time_range`** - _[string]_ - Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) +### `time_zones` - - **`comparison_mode`** - _[string]_ - Controls how to compare current data with historical or categorical baselines. Options: `none` (no comparison), `time` (compares with past based on default_time_range), `dimension` (compares based on comparison_dimension values) +_[array of string]_ - Refers to the time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) - - **`comparison_dimension`** - _[string]_ - for dimension mode, specify the comparison dimension by name +### `lock_time_zone` -### `embeds` +_[boolean]_ - When true, the dashboard will be locked to the first time provided in the time_zones list. When no time_zones are provided, the dashboard will be locked to UTC -_[object]_ - Configuration options for embedded dashboard views +### `allow_custom_time_range` - - **`hide_pivot`** - _[boolean]_ - When true, hides the pivot table view in embedded mode +_[boolean]_ - Defaults to true, when set to false it will hide the ability to set a custom time range for the user. -### `security` +### `defaults` -_[object]_ - Defines security rules and access control policies for resources +_[object]_ - defines the defaults YAML struct + ```yaml + defaults: #define all the defaults within here + dimensions: + - dim_1 + - dim_2 + measures: + - measure_1 + - measure_2 + time_range: P1M + comparison_mode: dimension #time, none + comparison_dimension: filename + ``` + - - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + - **`dimensions`** - _[anyOf]_ - Provides the default dimensions to load on viewing the dashboard - - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause + - **option 1** - _[string]_ - Simple field name as a string. - - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded + - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. - - **`if`** - _[string]_ - Expression to decide if the column should be included or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ + - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. - - **`names`** - _[anyOf]_ - List of fields to include. Should match the name of one of the dashboard's dimensions or measures _(required)_ + - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. - - **option 1** - _[array of string]_ - List of specific field names to include + - **`name`** - _[string]_ - Name of the field to select. _(required)_ - - **option 2** - _[string]_ - Wildcard '*' to include all fields + - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. - - **`exclude`** - _[array of object]_ - List of dimension or measure names to exclude from the dashboard. If exclude is defined all other dimensions and measures are included + - **`measures`** - _[anyOf]_ - Provides the default measures to load on viewing the dashboard - - **`if`** - _[string]_ - Expression to decide if the column should be excluded or not. It can leverage templated user attributes. Needs to be a valid SQL expression that evaluates to a boolean _(required)_ + - **option 1** - _[string]_ - Simple field name as a string. - - **`names`** - _[anyOf]_ - List of fields to exclude. Should match the name of one of the dashboard's dimensions or measures _(required)_ + - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. - - **option 1** - _[array of string]_ - List of specific field names to exclude + - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. - - **option 2** - _[string]_ - Wildcard '*' to exclude all fields + - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. - - **`rules`** - _[array of object]_ - List of detailed security rules that can be used to define complex access control policies + - **`name`** - _[string]_ - Name of the field to select. _(required)_ - - **`type`** - _[string]_ - Type of security rule - access (overall access), field_access (field-level access), or row_filter (row-level filtering) _(required)_ + - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. - - **`action`** - _[string]_ - Whether to allow or deny access for this rule + - **`time_range`** - _[string]_ - Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) - - **`if`** - _[string]_ - Conditional expression that determines when this rule applies. Must be a valid SQL expression that evaluates to a boolean + - **`comparison_mode`** - _[string]_ - Controls how to compare current data with historical or categorical baselines. Options: `none` (no comparison), `time` (compares with past based on default_time_range), `dimension` (compares based on comparison_dimension values) - - **`names`** - _[array of string]_ - List of field names this rule applies to (for field_access type rules) + - **`comparison_dimension`** - _[string]_ - for dimension mode, specify the comparison dimension by name - - **`all`** - _[boolean]_ - When true, applies the rule to all fields (for field_access type rules) +### `embeds` - - **`sql`** - _[string]_ - SQL expression for row filtering (for row_filter type rules) +_[object]_ - Configuration options for embedded dashboard views -## Common Properties + - **`hide_pivot`** - _[boolean]_ - When true, hides the pivot table view in embedded mode -### `name` +### `filters` -_[string]_ - Name is usually inferred from the filename, but can be specified manually. +_[array of object]_ - Default filters to apply to the dashboard -### `refs` + - **`dimension`** - _[string]_ - Dimension to filter on -_[array of string]_ - List of resource references + - **`value`** - _[no type]_ - Value to filter by -### `dev` + - **`operator`** - _[string]_ - Filter operator (eq, ne, in, etc.) -_[object]_ - Overrides any properties in development environment. +### `security` -### `prod` +_[object]_ - Defines security rules and access control policies for dashboards (without row filtering) -_[object]_ - Overrides any properties in production environment. \ No newline at end of file + - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. \ No newline at end of file diff --git a/docs/docs/hidden/yaml/metrics-views.md b/docs/docs/hidden/yaml/metrics-views.md index a6570193940..ae87481d88d 100644 --- a/docs/docs/hidden/yaml/metrics-views.md +++ b/docs/docs/hidden/yaml/metrics-views.md @@ -10,7 +10,7 @@ In your Rill project directory, create a metrics view, `.yaml`, fi ### `version` -_[string]_ - The version of the metrics view schema _(required)_ +_[string]_ - The version of the metrics view schema ### `type` @@ -30,7 +30,7 @@ _[string]_ - Extra instructions for AI agents. Used to guide natural language qu ### `model` -_[string]_ - Refers to the model powering the dashboard (either model or table is required) +_[string]_ - Refers to the model powering the dashboard (either model or table is required) _(required)_ ### `database` @@ -86,15 +86,17 @@ _[array of object]_ - Relates to exploring segments or dimensions of your data a _[array of object]_ - Used to define the numeric aggregates of columns from your data model - - **`name`** - _[string]_ - a stable identifier for the measure + - **`name`** - _[string]_ - a stable identifier for the measure _(required)_ - - **`display_name`** - _[string]_ - the display name of your measure. + - **`display_name`** - _[string]_ - the display name of your measure. _(required)_ - - **`description`** - _[string]_ - a freeform text description of the dimension + - **`label`** - _[string]_ - a label for your measure, deprecated use display_name + + - **`description`** - _[string]_ - a freeform text description of the measure - **`type`** - _[string]_ - Measure calculation type: "simple" for basic aggregations, "derived" for calculations using other measures, or "time_comparison" for period-over-period analysis. Defaults to "simple" unless dependencies exist. - - **`expression`** - _[string]_ - a combination of operators and functions for aggregations + - **`expression`** - _[string]_ - a combination of operators and functions for aggregations _(required)_ - **`window`** - _[anyOf]_ - A measure window can be defined as a keyword string (e.g. 'time' or 'all') or an object with detailed window configuration. For more information, see the [window functions](/build/metrics-view/advanced-expressions/windows) documentation. @@ -104,49 +106,15 @@ _[array of object]_ - Used to define the numeric aggregates of columns from your - **`partition`** - _[boolean]_ - Controls whether the window is partitioned. When true, calculations are performed within each partition separately. - - **`order`** - _[anyOf]_ - Specifies the fields to order the window by, determining the sequence of rows within each partition. - - - **option 1** - _[string]_ - Simple field name as a string. - - - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. - - - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. - - - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. - - - **`name`** - _[string]_ - Name of the field to select. _(required)_ - - - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. + - **`order`** - _[string]_ - Specifies the fields to order the window by, determining the sequence of rows within each partition. - **`frame`** - _[string]_ - Defines the window frame boundaries for calculations, specifying which rows are included in the window relative to the current row. - - **`per`** - _[anyOf]_ - for per dimensions - - - **option 1** - _[string]_ - Simple field name as a string. - - - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. - - - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. - - - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. - - - **`name`** - _[string]_ - Name of the field to select. _(required)_ - - - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. - - - **`requires`** - _[anyOf]_ - using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures - - - **option 1** - _[string]_ - Simple field name as a string. - - - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. + - **`per`** - _[string]_ - for per dimensions - - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. - - - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. + - **`requires`** - _[object]_ - using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures - - **`name`** - _[string]_ - Name of the field to select. _(required)_ - - - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. + - **`valid_percent_of_total`** - _[boolean]_ - a boolean indicating whether percent-of-total values should be rendered for this measure - **`format_preset`** - _[string]_ - Controls the formatting of this measure using a predefined preset. Measures cannot have both `format_preset` and `format_d3`. If neither is supplied, the measure will be formatted using the `humanize` preset by default. @@ -159,14 +127,41 @@ _[array of object]_ - Used to define the numeric aggregates of columns from your - `interval_ms`: Convert milliseconds into human-readable durations like hours (h), days (d), years (y), etc. (optional) + - **`format`** - _[string]_ - a custom format string for the measure + - **`format_d3`** - _[string]_ - Controls the formatting of this measure using a [d3-format](https://d3js.org/d3-format) string. If an invalid format string is supplied, the measure will fall back to `format_preset: humanize`. A measure cannot have both `format_preset` and `format_d3`. If neither is provided, the humanize preset is used by default. Example: `format_d3: ".2f"` formats using fixed-point notation with two decimal places. Example: `format_d3: ",.2r"` formats using grouped thousands with two significant digits. (optional) - - **`format_d3_locale`** - _[object]_ - locale configuration passed through to D3, enabling changing the currency symbol among other things. For details, see the docs for D3's [formatLocale](https://d3js.org/d3-format#formatLocale) + - **`format_d3_locale`** - _[object]_ - locale configuration passed through to D3, enabling changing the currency symbol among other things. For details, see the docs for D3's formatLocale. + ```yaml + format_d3: "$," + format_d3_locale: + grouping: [3, 2] + currency: ["₹", ""] + ``` + - - **`valid_percent_of_total`** - _[boolean]_ - a boolean indicating whether percent-of-total values should be rendered for this measure + - **`grouping`** - _[array]_ - the grouping of the currency symbol + + - **`currency`** - _[array]_ - the currency symbol + + - **`decimals`** - _[integer]_ - number of decimal places to display + + - **`prefix`** - _[string]_ - text to display before the measure value + + - **`suffix`** - _[string]_ - text to display after the measure value + + - **`multiplier`** - _[number]_ - multiply the measure value by this number + + - **`hidden`** - _[boolean]_ - if true, the measure will not be displayed in the UI - **`treat_nulls_as`** - _[string]_ - used to configure what value to fill in for missing time buckets. This also works generally as COALESCING over non empty time buckets. + - **`drill_through`** - _[object]_ - configuration for drill-through functionality + + - **`enabled`** - _[boolean]_ - whether drill-through is enabled for this measure + + - **`target`** - _[string]_ - the target dashboard or URL for drill-through + ### `annotations` _[array of object]_ - Used to define annotations that can be displayed on charts diff --git a/docs/docs/hidden/yaml/rillyaml.md b/docs/docs/hidden/yaml/rillyaml.md index bcb471d9f27..11908c021fe 100644 --- a/docs/docs/hidden/yaml/rillyaml.md +++ b/docs/docs/hidden/yaml/rillyaml.md @@ -197,6 +197,14 @@ mock_users: ## Common Properties +### `name` + +_[string]_ - Name is usually inferred from the filename, but can be specified manually. + +### `refs` + +_[array of string]_ - List of resource references + ### `dev` _[object]_ - Overrides any properties in development environment. diff --git a/docs/docs/hidden/yaml/sources.md b/docs/docs/hidden/yaml/sources.md index 3d5315350e0..04098885635 100644 --- a/docs/docs/hidden/yaml/sources.md +++ b/docs/docs/hidden/yaml/sources.md @@ -18,7 +18,7 @@ sidebar_position: 32 ### `type` -_[string]_ - Refers to the resource type and must be `model` _(required)_ +_[string]_ - Refers to the resource type and must be `connector` _(required)_ ### `connector` @@ -68,8 +68,11 @@ _[string]_ - The maximum time to wait for source ingestion ### `refresh` _[object]_ - Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying source data (optional). -- **cron** - a cron schedule expression, which should be encapsulated in single quotes, e.g. `* * * * *` (optional) -- **every** - a Go duration string, such as `24h` (optional) +```yaml +refresh: + cron: "* * * * *" + every: "24h" +``` - **`cron`** - _[string]_ - A cron schedule expression, which should be encapsulated in single quotes, e.g. `* * * * *` @@ -90,4 +93,22 @@ _[object]_ - Specifies the raw parameters to inject into the DuckDB read_csv, re ### `dsn` -_[string]_ - Used to set the Snowflake connection string \ No newline at end of file +_[string]_ - Used to set the Snowflake connection string + +## Common Properties + +### `name` + +_[string]_ - Name is usually inferred from the filename, but can be specified manually. + +### `refs` + +_[array of string]_ - List of resource references + +### `dev` + +_[object]_ - Overrides any properties in development environment. + +### `prod` + +_[object]_ - Overrides any properties in production environment. \ No newline at end of file diff --git a/docs/docs/hidden/yaml/themes.md b/docs/docs/hidden/yaml/themes.md index 664037f8393..381748ee361 100644 --- a/docs/docs/hidden/yaml/themes.md +++ b/docs/docs/hidden/yaml/themes.md @@ -4,10 +4,7 @@ title: Theme YAML sidebar_position: 40 --- -In your Rill project directory, create a `.yaml` file in any directory containing `type: theme`. Rill will automatically ingest the theme next time you run `rill start` or deploy to Rill Cloud. - -To apply that theme to a dashboard, add `default_theme: ` to the yaml file for that dashboard. Alternatively, you can add this to the end of the URL in your browser: `?theme=` - +Themes allow you to customize the appearance of your dashboards and UI components. ## Properties @@ -15,13 +12,39 @@ To apply that theme to a dashboard, add `default_theme: ` to the _[string]_ - Refers to the resource type and must be `theme` _(required)_ +### `display_name` + +_[string]_ - Display name for the theme _(required)_ + +### `description` + +_[string]_ - Description for the theme + ### `colors` -_[object]_ - Used to override the dashboard colors. Either primary or secondary color must be provided. _(required)_ +_[object]_ - Color palette for the theme + + - **`primary`** - _[string]_ - Primary color + + - **`secondary`** - _[string]_ - Secondary color + + - **`accent`** - _[string]_ - Accent color - - **`primary`** - _[string]_ - Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). + - **`background`** - _[string]_ - Background color - - **`secondary`** - _[string]_ - Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. + - **`text`** - _[string]_ - Text color + +### `fonts` + +_[object]_ - Font configuration for the theme + + - **`family`** - _[string]_ - Font family + + - **`size`** - _[string]_ - Base font size + +### `spacing` + +_[object]_ - Spacing configuration for the theme ## Common Properties @@ -39,14 +62,4 @@ _[object]_ - Overrides any properties in development environment. ### `prod` -_[object]_ - Overrides any properties in production environment. - -## Examples - -```yaml -# Example: You can copy this directly into your .yaml file -type: theme -colors: - primary: plum - secondary: violet -``` \ No newline at end of file +_[object]_ - Overrides any properties in production environment. \ No newline at end of file diff --git a/runtime/parser/schema/advanced-models.schema.yaml b/runtime/parser/old/advanced-models.schema.yaml similarity index 100% rename from runtime/parser/schema/advanced-models.schema.yaml rename to runtime/parser/old/advanced-models.schema.yaml diff --git a/runtime/parser/schema/alerts.schema.yaml b/runtime/parser/old/alerts.schema.yaml similarity index 100% rename from runtime/parser/schema/alerts.schema.yaml rename to runtime/parser/old/alerts.schema.yaml diff --git a/runtime/parser/schema/apis.schema.yaml b/runtime/parser/old/apis.schema.yaml similarity index 100% rename from runtime/parser/schema/apis.schema.yaml rename to runtime/parser/old/apis.schema.yaml diff --git a/runtime/parser/schema/canvas-dashboards.schema.yaml b/runtime/parser/old/canvas-dashboards.schema.yaml similarity index 100% rename from runtime/parser/schema/canvas-dashboards.schema.yaml rename to runtime/parser/old/canvas-dashboards.schema.yaml diff --git a/runtime/parser/schema/component.schema.yaml b/runtime/parser/old/component.schema.yaml similarity index 100% rename from runtime/parser/schema/component.schema.yaml rename to runtime/parser/old/component.schema.yaml diff --git a/runtime/parser/schema/connectors.schema.yaml b/runtime/parser/old/connectors.schema.yaml similarity index 100% rename from runtime/parser/schema/connectors.schema.yaml rename to runtime/parser/old/connectors.schema.yaml diff --git a/runtime/parser/schema/explore-dashboards.schema.yaml b/runtime/parser/old/explore-dashboards.schema.yaml similarity index 100% rename from runtime/parser/schema/explore-dashboards.schema.yaml rename to runtime/parser/old/explore-dashboards.schema.yaml diff --git a/runtime/parser/schema/metrics_views.schema.yaml b/runtime/parser/old/metrics_views.schema.yaml similarity index 100% rename from runtime/parser/schema/metrics_views.schema.yaml rename to runtime/parser/old/metrics_views.schema.yaml diff --git a/runtime/parser/schema/models.schema.yaml b/runtime/parser/old/models.schema.yaml similarity index 100% rename from runtime/parser/schema/models.schema.yaml rename to runtime/parser/old/models.schema.yaml diff --git a/runtime/parser/schema/rillyaml.schema.yaml b/runtime/parser/old/rillyaml.schema.yaml similarity index 100% rename from runtime/parser/schema/rillyaml.schema.yaml rename to runtime/parser/old/rillyaml.schema.yaml diff --git a/runtime/parser/schema/sources.schema.yaml b/runtime/parser/old/sources.schema.yaml similarity index 100% rename from runtime/parser/schema/sources.schema.yaml rename to runtime/parser/old/sources.schema.yaml diff --git a/runtime/parser/schema/themes.schema.yaml b/runtime/parser/old/themes.schema.yaml similarity index 100% rename from runtime/parser/schema/themes.schema.yaml rename to runtime/parser/old/themes.schema.yaml diff --git a/runtime/parser/schema/project.schema.yaml b/runtime/parser/schema/project.schema.yaml index fa2fb766b03..8fbae53a33e 100644 --- a/runtime/parser/schema/project.schema.yaml +++ b/runtime/parser/schema/project.schema.yaml @@ -21,19 +21,1433 @@ description: | ::: oneOf: - - $ref: 'connectors.schema.yaml#' - - $ref: 'sources.schema.yaml#' - - $ref: 'models.schema.yaml#' - - $ref: 'advanced-models.schema.yaml#' - - $ref: 'metrics_views.schema.yaml#' - - $ref: 'canvas-dashboards.schema.yaml#' - # - $ref: 'component.schema.yaml#' - - $ref: 'explore-dashboards.schema.yaml#' - - - - $ref: 'alerts.schema.yaml#' - - $ref: 'apis.schema.yaml#' - - $ref: 'themes.schema.yaml#' + # Connector YAML + - title: Connector YAML + id: connectors + type: object + description: | + Connector YAML files define how Rill connects to external data sources and OLAP engines. Each connector specifies a driver type and its required connection parameters. + + ## Available Connector Types + + ### _OLAP Engines_ + - [**DuckDB**](#duckdb) - Embedded DuckDB engine (default) + - [**ClickHouse**](#clickhouse) - ClickHouse analytical database + - [**MotherDuck**](#motherduck) - MotherDuck cloud database + - [**Druid**](#druid) - Apache Druid + - [**Pinot**](#pinot) - Apache Pinot + + ### _Data Warehouses_ + - [**Snowflake**](#snowflake) - Snowflake data warehouse + - [**BigQuery**](#bigquery) - Google BigQuery + - [**Redshift**](#redshift) - Amazon Redshift + - [**Athena**](#athena) - Amazon Athena + + ### _Databases_ + - [**PostgreSQL**](#postgres) - PostgreSQL databases + - [**MySQL**](#mysql) - MySQL databases + - [**SQLite**](#sqlite) - SQLite databases + + ### _Cloud Storage_ + - [**GCS**](#gcs) - Google Cloud Storage + - [**S3**](#s3) - Amazon S3 storage + - [**Azure**](#azure) - Azure Blob Storage + + ### _Other_ + - [**HTTPS**](#https) - Public files via HTTP/HTTPS + - [**Salesforce**](#salesforce) - Salesforce data + - [**Slack**](#slack) - Slack data + + :::warning Security Recommendation + For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env.connector..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/build/credentials/) for complete setup instructions. + ::: + allOf: + - title: Properties + type: object + properties: + type: + type: string + const: connector + description: Refers to the resource type and must be `connector` + required: + - type + + + - $ref: '#/definitions/common_properties' + + - oneOf: + - $ref: '#/definitions/athena' + - $ref: '#/definitions/azure' + - $ref: '#/definitions/bigquery' + - $ref: '#/definitions/clickhouse' + - $ref: '#/definitions/druid' + - $ref: '#/definitions/duckdb' + - $ref: '#/definitions/gcs' + - $ref: '#/definitions/https' + - $ref: '#/definitions/motherduck' + - $ref: '#/definitions/mysql' + - $ref: '#/definitions/pinot' + - $ref: '#/definitions/postgres' + - $ref: '#/definitions/redshift' + - $ref: '#/definitions/s3' + - $ref: '#/definitions/salesforce' + - $ref: '#/definitions/slack' + - $ref: '#/definitions/snowflake' + - $ref: '#/definitions/sqlite' + + # Source YAML + - title: Source YAML + type: object + id: sources + description: | + :::warning Deprecated Feature + **Sources have been deprecated** and are now considered "source models." While sources remain backward compatible, we recommend migrating to the new source model format for access to the latest features and improvements. + + **Next steps:** + - Continue using sources if needed (backward compatible) + - Migrate to source models via the `type:model` parameter for existing projects + - See our [model YAML reference](advanced-models) for current documentation and best practices + ::: + allOf: + - title: Properties + type: object + properties: + type: + type: string + const: connector + description: Refers to the resource type and must be `connector` + connector: + type: string + description: Refers to the connector type for the source, see [connectors](/reference/project-files/connectors) for more information + enum: + - https + - s3 + - gcs + - local_file + - motherduck + - athena + - redshift + - postgres + - sqlite + - snowflake + - bigquery + - duckdb + uri: + type: string + description: | + Refers to the URI of the remote connector you are using for the source. Rill also supports glob patterns as part of the URI for S3 and GCS (required for type: http, s3, gcs). + + - `s3://your-org/bucket/file.parquet` — the s3 URI of your file + - `gs://your-org/bucket/file.parquet` — the gsutil URI of your file + - `https://data.example.org/path/to/file.parquet` — the web address of your file + path: + type: string + description: Refers to the local path of the connector you are using for the source + sql: + type: string + description: Sets the SQL query to extract data from a SQL source + region: + type: string + description: Sets the cloud region of the S3 bucket or Athena + endpoint: + type: string + description: Overrides the S3 endpoint to connect to + output_location: + type: string + description: Sets the query output location and result files in Athena + workgroup: + type: string + description: Sets a workgroup for Athena connector + project_id: + type: string + description: Sets a project id to be used to run BigQuery jobs + timeout: + type: string + description: The maximum time to wait for source ingestion + refresh: + type: object + description: | + Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying source data (optional). + ```yaml + refresh: + cron: "* * * * *" + every: "24h" + ``` + properties: + cron: + type: string + description: A cron schedule expression, which should be encapsulated in single quotes, e.g. `* * * * *` + every: + type: string + description: A Go duration string, such as `24h` + db: + type: string + description: Sets the database for motherduck connections and/or the path to the DuckDB/SQLite db file + database_url: + type: string + description: Postgres connection string that should be used + duckdb: + type: object + description: Specifies the raw parameters to inject into the DuckDB read_csv, read_json or read_parquet statement + additionalProperties: true + dsn: + type: string + description: Used to set the Snowflake connection string + required: + - type + - connector + - $ref: '#/definitions/common_properties' + # Model SQL + - title: Model SQL + type: object + id: models + description: | + When using Rill Developer, data transformations are powered by DuckDB and their dialect of SQL. Under the hood, by default, data models are created as views in DuckDB. Please check our modeling page and DuckDB documentation for more details about how to construct and write your model SQL syntax. + + In your Rill project directory, you can also create a `.sql` file containing an appropriate DuckDB `SELECT` statement, most commonly within the default `models` directory, to represent a model (or set of SQL transformations). Rill will automatically detect and parse the model next time you run `rill start`. + + ### Annotating your models with properties + In most cases, objects are represented in Rill as YAML files. Models are unique in that any model.sql file can be considered a model resource in Rill, representing a SQL transformation that you would like to inform using a set of inputs and outputting a view or table (depending on the materialization type). For most other resources, available properties can be set directly via the corresponding YAML file. In the case of a model SQL file though, configurable properties should be set by annotating the top of the file using the following syntax: + ```sql + -- @property: value + ``` + We will cover different available configurable properties in the below sections. + allOf: + - title: Properties + type: object + properties: + type: + type: string + const: model + description: | + By default, any new model that is created in a Rill project will populate a corresponding .sql file representing the model. Similarly, a .sql file that is directly created in the project directory will also be automatically assumed by Rill to be a model by default. Therefore, it is not necessary to annotate the model resource with the type property. + + For consistency or documentation purposes, if you'd like to annotate your model resource as well with the type property, you can do so by adding the following to the top of your model_name.sql: + ```sql + -- @type: model + ``` + materialize: + type: boolean + description: | + As mentioned, models will be materialized in DuckDB as views by default. However, you can choose to materialize them as tables instead of views. To do this, you can add the following annotation to the top of your model SQL file: + ```sql + -- @materialize: true + ``` + + Alternatively, it is possible to set it as a project-wide default as well that your models inherit via your rill.yaml file: + ```yaml + models: + materialize: true + ``` + + :::info To materialize or not to materialize? + + There are both pros and cons to materializing your models. + - Pros can include improved performance for downstream models and dashboards, especially with the SQL is complex and/or the data size is large. We generally recommend _materializing_ final models that power dashboards. + - Cons can include a degraded keystroke-by-keystroke modeling experience or for specific edge cases, such as when using cross joins. + + If unsure, we would generally recommend leaving the defaults and/or reaching out for further guidance! + ::: + + # Advanced Models + - title: Models YAML + id: advanced-models + type: object + description: | + :::tip + + Both regular models and source models can use the Model YAML specification described on this page. While [SQL models](./models) are perfect for simple transformations, Model YAML files provide advanced capabilities for complex data processing scenarios. + + **When to use Model YAML:** + - **Partitions** - Optimize performance with data partitioning strategies + - **Incremental models** - Process only new or changed data efficiently + - **Pre/post execution hooks** - Run custom logic before or after model execution + - **Staging** - Create intermediate tables for complex transformations + - **Output configuration** - Define specific output formats and destinations + + Model YAML files give you fine-grained control over how your data is processed and transformed, making them ideal for production workloads and complex analytics pipelines. + + ::: + allOf: + - title: Properties + type: object + properties: + type: + type: string + const: model + description: Refers to the resource type and must be `model` + refresh: + $ref: '#/definitions/schedule_properties' + description: | + Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying model data + ```yaml + refresh: + cron: "* * * * *" + #every: "24h" + ``` + + connector: + type: string + const: connector + description: Refers to the resource type and is needed if setting an explicit OLAP engine. IE `clickhouse` + sql: + type: string + description: Raw SQL query to run against source + timeout: + type: string + description: The maximum time to wait for model ingestion + incremental: + type: boolean + description: whether incremental modeling is required (optional) + change_mode: + type: string + enum: + - reset + - manual + - patch + description: Configure how changes to the model specifications are applied (optional). 'reset' will drop and recreate the model automatically, 'manual' will require a manual full or incremental refresh to apply changes, and 'patch' will switch to the new logic without re-processing historical data (only applies for incremental models). + state: + $ref: '#/definitions/data_properties' + description: Refers to the explicitly defined state of your model, cannot be used with partitions (optional) + partitions: + $ref: '#/definitions/data_properties' + description: Refers to the how your data is partitioned, cannot be used with state. (optional) + materialize: + type: boolean + description: models will be materialized in olap + partitions_watermark: + type: string + description: Refers to a customizable timestamp that can be set to check if an object has been updated (optional). + partitions_concurrency: + type: integer + description: Refers to the number of concurrent partitions that can be read at the same time (optional). + stage: + type: object + properties: + connector: + type: string + description: Refers to the connector type for the staging table + required: + - connector + description: in the case of staging models, where an input source does not support direct write to the output and a staging table is required + additionalProperties: true + output: + type: object + description: to define the properties of output + properties: + table: + type: string + description: Name of the output table. If not specified, the model name is used. + materialize: + type: boolean + description: Whether to materialize the model as a table or view + connector: + type: string + description: Refers to the connector type for the output table. Can be `clickhouse` or `duckdb` and their named connector + incremental_strategy: + type: string + enum: + - append + - merge + - partition_overwrite + description: Strategy to use for incremental updates. Can be 'append', 'merge' or 'partition_overwrite' + unique_key: + type: array + items: + type: string + description: List of columns that uniquely identify a row for merge strategy + partition_by: + type: string + description: Column or expression to partition the table by + required: + - type + - sql + - $ref: '#/definitions/common_properties' + + # Metrics Views + - title: Metrics View YAML + id: metrics-views + type: object + description: In your Rill project directory, create a metrics view, `.yaml`, file in the `metrics` directory. Rill will ingest the metric view definition next time you run `rill start`. + allOf: + - title: Properties + type: object + properties: + version: + type: string + description: The version of the metrics view schema + type: + type: string + const: metrics_view + description: Refers to the resource type and must be `metrics_view` + display_name: + type: string + description: Refers to the display name for the metrics view + description: + type: string + description: Refers to the description for the metrics view + ai_instructions: + type: string + description: Extra instructions for AI agents. Used to guide natural language question answering and routing. + model: + type: string + description: Refers to the model powering the dashboard (either model or table is required) + database: + type: string + description: Refers to the database to use in the OLAP engine (to be used in conjunction with table). Otherwise, will use the default database or schema if not specified + database_schema: + type: string + description: Refers to the schema to use in the OLAP engine (to be used in conjunction with table). Otherwise, will use the default database or schema if not specified + table: + type: string + description: Refers to the table powering the dashboard, should be used instead of model for dashboards create from external OLAP tables (either table or model is required) + timeseries: + type: string + description: Refers to the timestamp column from your model that will underlie x-axis data in the line charts. If not specified, the line charts will not appear + watermark: + type: string + description: A SQL expression that tells us the max timestamp that the metrics are considered valid for. Usually does not need to be overwritten + smallest_time_grain: + type: string + description: 'Refers to the smallest time granularity the user is allowed to view. The valid values are: millisecond, second, minute, hour, day, week, month, quarter, year' + first_day_of_week: + type: integer + description: Refers to the first day of the week for time grain aggregation (for example, Sunday instead of Monday). The valid values are 1 through 7 where Monday=1 and Sunday=7 + first_month_of_year: + type: integer + description: Refers to the first month of the year for time grain aggregation. The valid values are 1 through 12 where January=1 and December=12 + dimensions: + type: array + description: Relates to exploring segments or dimensions of your data and filtering the dashboard + items: + type: object + properties: + name: + type: string + description: a stable identifier for the dimension + display_name: + type: string + description: a display name for your dimension + description: + type: string + description: a freeform text description of the dimension + column: + type: string + description: a categorical column + expression: + type: string + description: a non-aggregate expression such as string_split(domain, '.'). One of column and expression is required but cannot have both at the same time + unnest: + type: boolean + description: if true, allows multi-valued dimension to be unnested (such as lists) and filters will automatically switch to "contains" instead of exact match + uri: + type: + - string + - boolean + description: enable if your dimension is a clickable URL to enable single click navigation (boolean or valid SQL expression) + anyOf: + - required: + - column + - required: + - expression + measures: + type: array + description: Used to define the numeric aggregates of columns from your data model + items: + type: object + properties: + name: + type: string + description: a stable identifier for the measure + display_name: + type: string + description: the display name of your measure. + label: + type: string + description: a label for your measure, deprecated use display_name + description: + type: string + description: a freeform text description of the measure + type: + type: string + description: 'Measure calculation type: "simple" for basic aggregations, "derived" for calculations using other measures, or "time_comparison" for period-over-period analysis. Defaults to "simple" unless dependencies exist.' + expression: + type: string + description: a combination of operators and functions for aggregations + window: + description: A measure window can be defined as a keyword string (e.g. 'time' or 'all') or an object with detailed window configuration. For more information, see the [window functions](/build/metrics-view/advanced-expressions/windows) documentation. + anyOf: + - type: string + enum: + - time + - 'true' + - all + description: 'Shorthand: `time` or `true` means time-partitioned, `all` means non-partitioned.' + - type: object + description: 'Detailed window configuration for measure calculations, allowing control over partitioning, ordering, and frame definition.' + properties: + partition: + type: boolean + description: 'Controls whether the window is partitioned. When true, calculations are performed within each partition separately.' + order: + type: string + description: 'Specifies the fields to order the window by, determining the sequence of rows within each partition.' + frame: + type: string + description: 'Defines the window frame boundaries for calculations, specifying which rows are included in the window relative to the current row.' + additionalProperties: false + per: + type: string + description: for per dimensions + requires: + type: object + description: using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures + additionalProperties: true + valid_percent_of_total: + type: boolean + description: a boolean indicating whether percent-of-total values should be rendered for this measure + format_preset: + type: string + description: | + Controls the formatting of this measure using a predefined preset. Measures cannot have both `format_preset` and `format_d3`. If neither is supplied, the measure will be formatted using the `humanize` preset by default. + + Available options: + - `humanize`: Round numbers into thousands (K), millions(M), billions (B), etc. + - `none`: Raw output. + - `currency_usd`: Round to 2 decimal points with a dollar sign ($). + - `currency_eur`: Round to 2 decimal points with a euro sign (€). + - `percentage`: Convert a rate into a percentage with a % sign. + - `interval_ms`: Convert milliseconds into human-readable durations like hours (h), days (d), years (y), etc. (optional) + format: + type: string + description: a custom format string for the measure + format_d3: + type: string + description: 'Controls the formatting of this measure using a [d3-format](https://d3js.org/d3-format) string. If an invalid format string is supplied, the measure will fall back to `format_preset: humanize`. A measure cannot have both `format_preset` and `format_d3`. If neither is provided, the humanize preset is used by default. Example: `format_d3: ".2f"` formats using fixed-point notation with two decimal places. Example: `format_d3: ",.2r"` formats using grouped thousands with two significant digits. (optional)' + format_d3_locale: + type: object + description: | + locale configuration passed through to D3, enabling changing the currency symbol among other things. For details, see the docs for D3's formatLocale. + ```yaml + format_d3: "$," + format_d3_locale: + grouping: [3, 2] + currency: ["₹", ""] + ``` + properties: + grouping: + type: array + description: the grouping of the currency symbol + currency: + type: array + description: the currency symbol + decimals: + type: integer + description: number of decimal places to display + prefix: + type: string + description: text to display before the measure value + suffix: + type: string + description: text to display after the measure value + multiplier: + type: number + description: multiply the measure value by this number + hidden: + type: boolean + description: if true, the measure will not be displayed in the UI + treat_nulls_as: + type: string + description: used to configure what value to fill in for missing time buckets. This also works generally as COALESCING over non empty time buckets. + drill_through: + type: object + description: configuration for drill-through functionality + properties: + enabled: + type: boolean + description: whether drill-through is enabled for this measure + target: + type: string + description: the target dashboard or URL for drill-through + required: + - name + - display_name + - expression + + annotations: + type: array + description: Used to define annotations that can be displayed on charts + items: + type: object + properties: + name: + type: string + description: A stable identifier for the annotation. Defaults to model or table names when not specified + model: + type: string + description: Refers to the model powering the annotation (either table or model is required). The model must have 'time' and 'description' columns. Optional columns include 'time_end' for range annotations and 'grain' to specify when the annotation should appear based on dashboard grain level. + database: + type: string + description: Refers to the database to use in the OLAP engine (to be used in conjunction with table). Otherwise, will use the default database or schema if not specified + database_schema: + type: string + description: Refers to the schema to use in the OLAP engine (to be used in conjunction with table). Otherwise, will use the default database or schema if not specified + table: + type: string + description: Refers to the table powering the annotation, should be used instead of model for annotations from external OLAP tables (either table or model is required) + connector: + type: string + description: Refers to the connector to use for the annotation + measures: + description: Specifies which measures to apply the annotation to. Applies to all measures if not specified + anyOf: + - type: string + description: Simple field name as a string. + - type: array + description: List of field selectors, each can be a string or an object with detailed configuration. + items: + anyOf: + - type: string + description: Shorthand field selector, interpreted as the name. + - type: object + description: Detailed field selector configuration with name and optional time grain. + properties: + name: + type: string + description: Name of the field to select. + time_grain: + type: string + description: Time grain for time-based dimensions. + enum: + - '' + - ms + - millisecond + - s + - second + - min + - minute + - h + - hour + - d + - day + - w + - week + - month + - q + - quarter + - 'y' + - year + required: + - name + additionalProperties: fal + security: + $ref: '#/definitions/security_policy_properties' + description: Defines a security policy for the dashboard + required: + - type + - model + + - $ref: '#/definitions/common_properties' + + # Canvas Dashboards + - title: Canvas Dashboard YAML + id: canvas-dashboards + type: object + description: Canvas dashboards provide a flexible way to create custom dashboards with drag-and-drop components. + allOf: + - title: Properties + type: object + properties: + type: + type: string + const: canvas + description: Refers to the resource type and must be `canvas` + display_name: + type: string + description: Refers to the display name for the canvas + description: + type: string + description: Description for the canvas dashboard + banner: + type: string + description: Refers to the custom banner displayed at the header of an Canvas dashboard + rows: + type: array + description: Refers to all of the rows displayed on the Canvas + items: + type: object + properties: + height: + type: string + description: Height of the row in px + items: + type: array + description: List of components to display in the row + items: + type: object + properties: + component: + type: string + description: | + Name of the component to display. Each component type has its own set of properties. + Available component types: + + - **markdown** - Text component, uses markdown formatting + - **kpi_grid** - KPI component, similar to TDD in Rill Explore, display quick KPI charts + - **stacked_bar_normalized** - Bar chart normalized to 100% values + - **line_chart** - Normal Line chart + - **bar_chart** - Normal Bar chart + - **stacked_bar** - Stacked Bar chart + - **area_chart** - Line chart with area + - **image** - Provide a URL to embed into canvas dashboard + - **table** - Similar to Pivot table, add dimensions and measures to visualize your data + - **heatmap** - Heat Map chart to visualize distribution of data + - **donut_chart** - Donut or Pie chart to display sums of total + width: + type: + - string + - integer + description: Width of the component (can be a number or string with unit) + additionalProperties: true + additionalProperties: false + components: + type: array + description: Array of components to display on the canvas + items: + type: object + properties: + type: + type: string + description: Type of component (chart, table, text, etc.) + title: + type: string + description: Title for the component + data: + $ref: '#/definitions/data_properties' + description: Data source for the component + layout: + type: object + description: Layout configuration for the component + additionalProperties: true + style: + type: object + description: Styling configuration for the component + additionalProperties: true + max_width: + type: integer + description: Max width in pixels of the canvas + minimum: 0 + gap_x: + type: integer + description: Horizontal gap in pixels of the canvas + minimum: 0 + gap_y: + type: integer + description: Vertical gap in pixels of the canvas + minimum: 0 + theme: + oneOf: + - title: Existing theme + type: string + description: Name of an existing theme to apply to the dashboard + - title: Inline theme + type: object + description: Inline theme configuration. + description: Theme configuration. Can be either a string reference to an existing theme or an inline theme configuration object. + allow_custom_time_range: + type: boolean + description: Defaults to true, when set to false it will hide the ability to set a custom time range for the user. + time_ranges: + type: array + description: | + Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' + ```yaml + time_ranges: + - PT15M // Simplified syntax to specify only the range + - PT1H + - PT6H + - P7D + - range: P5D // Advanced syntax to specify comparison_offsets as well + - P4W + - rill-TD // Today + - rill-WTD // Week-To-date + ``` + items: + $ref: '#/definitions/explore_time_range_properties' + time_zones: + type: array + description: Refers to the time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) + items: + type: string + filters: + type: object + description: Indicates if filters should be enabled for the canvas. + additionalProperties: true + security: + description: Security rules to apply for access to the canvas dashboard + $ref: '#/definitions/dashboard_security_policy_properties' + required: + - type + - display_name + + # Explore Dashboards + - title: Explore Dashboard YAML + id: explore-dashboards + type: object + description: Explore dashboards provide an interactive way to explore data with predefined metrics and dimensions. + allOf: + - title: Properties + type: object + properties: + type: + type: string + const: explore + description: Refers to the resource type and must be `explore` + display_name: + type: string + description: Refers to the display name for the explore dashboard + metrics_view: + type: string + description: Refers to the metrics view resource + description: + type: string + description: Refers to the description of the explore dashboard + banner: + type: string + description: Refers to the custom banner displayed at the header of an explore dashboard + dimensions: + description: | + List of dimension names. Use '*' to select all dimensions (default) + ```yaml + # Example: Select a dimension + dimensions: + - country + + # Example: Select all dimensions except one + dimensions: + exclude: + - country + + # Example: Select all dimensions that match a regex + dimensions: + regex: "^public_.*$" + ``` + anyOf: + - type: string + description: Simple field name as a string. + - type: array + description: List of field selectors, each can be a string or an object with detailed configuration. + items: + anyOf: + - type: string + description: Shorthand field selector, interpreted as the name. + - type: object + description: Detailed field selector configuration with name and optional time grain. + properties: + name: + type: string + description: Name of the field to select. + time_grain: + type: string + description: Time grain for time-based dimensions. + enum: + - '' + - ms + - millisecond + - s + - second + - min + - minute + - h + - hour + - d + - day + - w + - week + - month + - q + - quarter + - 'y' + - year + required: + - name + additionalProperties: false + measures: + description: List of measure names. Use '*' to select all measures (default) + anyOf: + - type: string + description: Simple field name as a string. + - type: array + description: List of field selectors, each can be a string or an object with detailed configuration. + items: + anyOf: + - type: string + description: Shorthand field selector, interpreted as the name. + - type: object + description: Detailed field selector configuration with name and optional time grain. + properties: + name: + type: string + description: Name of the field to select. + time_grain: + type: string + description: Time grain for time-based dimensions. + enum: + - '' + - ms + - millisecond + - s + - second + - min + - minute + - h + - hour + - d + - day + - w + - week + - month + - q + - quarter + - 'y' + - year + required: + - name + additionalProperties: false + theme: + oneOf: + - title: Existing theme + type: string + description: Name of an existing theme to apply to the dashboard + - title: Inline theme + type: object + description: Inline theme configuration. + description: Name of the theme to use. Only one of theme and embedded_theme can be set. + time_range: + $ref: '#/definitions/explore_time_range_properties' + description: Default time range for the dashboard + time_ranges: + type: array + description: | + Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' + ```yaml + time_ranges: + - PT15M // Simplified syntax to specify only the range + - PT1H + - PT6H + - P7D + - range: P5D // Advanced syntax to specify comparison_offsets as well + - P4W + - rill-TD // Today + - rill-WTD // Week-To-date + ``` + items: + $ref: '#/definitions/explore_time_range_properties' + time_zones: + type: array + description: Refers to the time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) + items: + type: string + lock_time_zone: + type: boolean + description: When true, the dashboard will be locked to the first time provided in the time_zones list. When no time_zones are provided, the dashboard will be locked to UTC + allow_custom_time_range: + type: boolean + description: Defaults to true, when set to false it will hide the ability to set a custom time range for the user. + defaults: + type: object + description: | + defines the defaults YAML struct + ```yaml + defaults: #define all the defaults within here + dimensions: + - dim_1 + - dim_2 + measures: + - measure_1 + - measure_2 + time_range: P1M + comparison_mode: dimension #time, none + comparison_dimension: filename + ``` + properties: + dimensions: + description: Provides the default dimensions to load on viewing the dashboard + anyOf: + - type: string + description: Simple field name as a string. + - type: array + description: List of field selectors, each can be a string or an object with detailed configuration. + items: + anyOf: + - type: string + description: Shorthand field selector, interpreted as the name. + - type: object + description: Detailed field selector configuration with name and optional time grain. + properties: + name: + type: string + description: Name of the field to select. + time_grain: + type: string + description: Time grain for time-based dimensions. + enum: + - '' + - ms + - millisecond + - s + - second + - min + - minute + - h + - hour + - d + - day + - w + - week + - month + - q + - quarter + - 'y' + - year + required: + - name + additionalProperties: false + measures: + description: Provides the default measures to load on viewing the dashboard + anyOf: + - type: string + description: Simple field name as a string. + - type: array + description: List of field selectors, each can be a string or an object with detailed configuration. + items: + anyOf: + - type: string + description: Shorthand field selector, interpreted as the name. + - type: object + description: Detailed field selector configuration with name and optional time grain. + properties: + name: + type: string + description: Name of the field to select. + time_grain: + type: string + description: Time grain for time-based dimensions. + enum: + - '' + - ms + - millisecond + - s + - second + - min + - minute + - h + - hour + - d + - day + - w + - week + - month + - q + - quarter + - 'y' + - year + required: + - name + additionalProperties: false + time_range: + description: Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + type: string + comparison_mode: + description: 'Controls how to compare current data with historical or categorical baselines. Options: `none` (no comparison), `time` (compares with past based on default_time_range), `dimension` (compares based on comparison_dimension values)' + type: string + comparison_dimension: + description: 'for dimension mode, specify the comparison dimension by name' + type: string + additionalProperties: false + embeds: + type: object + description: Configuration options for embedded dashboard views + properties: + hide_pivot: + type: boolean + description: When true, hides the pivot table view in embedded mode + additionalProperties: false + filters: + type: array + description: Default filters to apply to the dashboard + items: + type: object + properties: + dimension: + type: string + description: Dimension to filter on + value: + description: Value to filter by + operator: + type: string + description: Filter operator (eq, ne, in, etc.) + security: + description: Security rules to apply for access to the explore dashboard + $ref: '#/definitions/dashboard_security_policy_properties' + required: + - type + - display_name + - metrics_view + + # Alerts + - title: Alert YAML + id: alerts + type: object + description: Along with alertings at the dashboard level and can be created via the UI, there might be more extensive alerting that you might want to develop and can be done so the an alert.yaml. When creating an alert via a YAML file, you'll see this denoted in the UI as `Created through code`. + allOf: + - title: Properties + type: object + properties: + type: + type: string + const: alert + description: Refers to the resource type and must be `alert` + refresh: + $ref: '#/definitions/schedule_properties' + description: | + Refresh schedule for the alert + ```yaml + refresh: + cron: "* * * * *" + #every: "24h" + ``` + display_name: + type: string + description: Display name for the alert + description: + type: string + description: Description for the alert + data: + $ref: '#/definitions/data_properties' + description: Data source for the alert + condition: + type: object + description: Condition that triggers the alert + properties: + operator: + type: string + description: Comparison operator (gt, lt, eq, etc.) + threshold: + description: Threshold value for the condition + measure: + type: string + description: Measure to compare against the threshold + notify: + $ref: '#/definitions/notify_properties' + description: Notification configuration + + required: + - type + - display_name + - data + - condition + - notify + - $ref: '#/definitions/common_properties' + + # APIs + - title: API YAML + id: apis + type: object + description: Custom APIs allow you to create endpoints that can be called to retrieve or manipulate data. + allOf: + - title: Properties + type: object + properties: + type: + type: string + const: api + description: Refers to the resource type and must be `api` + openapi: + type: object + description: OpenAPI specification for the API endpoint + properties: + summary: + type: string + description: A brief description of what the API endpoint does + sample: "Get user analytics data" + parameters: + type: array + description: List of parameters that the API endpoint accepts + items: + type: object + additionalProperties: true + request_schema: + type: object + description: JSON schema for the request body (use nested YAML instead of a JSON string) + additionalProperties: true + response_schema: + type: object + description: JSON schema for the response body (use nested YAML instead of a JSON string) + additionalProperties: true + security: + $ref: '#/definitions/security_policy_properties' + description: Security configuration for the API + skip_nested_security: + type: boolean + description: Flag to control security inheritance + sample: false + required: + - type + - $ref: '#/definitions/data_properties' + - $ref: '#/definitions/common_properties' + + # Themes + - title: Theme YAML + id: themes + type: object + description: Themes allow you to customize the appearance of your dashboards and UI components. + allOf: + - title: Properties + type: object + properties: + type: + type: string + const: theme + description: Refers to the resource type and must be `theme` + display_name: + type: string + description: Display name for the theme + description: + type: string + description: Description for the theme + colors: + type: object + description: Color palette for the theme + properties: + primary: + type: string + description: Primary color + secondary: + type: string + description: Secondary color + accent: + type: string + description: Accent color + background: + type: string + description: Background color + text: + type: string + description: Text color + additionalProperties: true + fonts: + type: object + description: Font configuration for the theme + properties: + family: + type: string + description: Font family + size: + type: string + description: Base font size + additionalProperties: true + spacing: + type: object + description: Spacing configuration for the theme + additionalProperties: true + required: + - type + - display_name + - $ref: '#/definitions/common_properties' + + # Rill YAML + - title: Project YAML + id: rillyaml + type: object + description: The `rill.yaml` file contains metadata about your project. + allOf: + - title: Properties + type: object + properties: + compiler: + type: string + description: Specifies the parser version to use for compiling resources + display_name: + type: string + description: The display name of the project, shown in the upper-left corner of the UI + description: + type: string + description: A brief description of the project + features: + type: object + description: Optional feature flags. Can be specified as a map of feature names to booleans. + ai_instructions: + type: string + description: Extra instructions for LLM/AI features. Used to guide natural language question answering and routing. + - title: Configuring the default OLAP Engine + description: | + Rill allows you to specify the default OLAP engine to use in your project via `rill.yaml`. + :::info Curious about OLAP Engines? + Please see our reference documentation on [OLAP Engines](/docs/reference/olap-engines/olap-engines.md). + ::: + type: object + properties: + olap_connector: + type: string + description: Specifies the default OLAP engine for the project. Defaults to duckdb if not set. + examples: + - olap_connector: clickhouse + - title: Project-wide defaults + type: object + description: | + In `rill.yaml`, project-wide defaults can be specified for a resource type within a project. Unless otherwise specified, _individual resources will inherit any defaults_ that have been specified in `rill.yaml`. For available properties that can be configured, please refer to the YAML specification for each individual resource type - [model](advanced-models.md), [metrics_view](metrics-views.md), and [explore](explore-dashboards.md) + + :::note Use plurals when specifying project-wide defaults + In your `rill.yaml`, the top level property for the resource type needs to be **plural**, such as `models`, `metrics_views` and `explores`. + ::: + + :::info Hierarchy of inheritance and property overrides + As a general rule of thumb, properties that have been specified at a more _granular_ level will supercede or override higher level properties that have been inherited. Therefore, in order of inheritance, Rill will prioritize properties in the following order: + 1. Individual [models](advanced-models.md)/[metrics_views](metrics-views.md)/[explore](explore-dashboards.md) object level properties (e.g. `advanced-models.yaml` or `explore-dashboards.yaml`) + 2. [Environment](/docs/build/models/environments.md) level properties (e.g. a specific property that have been set for `dev`) + 3. [Project-wide defaults](#project-wide-defaults) for a specific property and resource type + ::: + properties: + models: + type: object + description: Defines project-wide default settings for models. Unless overridden, individual models will inherit these defaults. + metrics_views: + type: object + description: Defines project-wide default settings for metrics_views. Unless overridden, individual metrics_views will inherit these defaults. + explores: + type: object + description: Defines project-wide default settings for explores. Unless overridden, individual explores will inherit these defaults. + examples: + - # For example, the following YAML configuration below will set a project-wide default for: + # Models - Configure a [source refresh](/build/connect/source-refresh.md). + # Metrics View - Set the [first day of the week](metrics-view.md) for timeseries aggregations to be Sunday along with setting the smallest_time_grain. + # Explore Dashboards - Set the [default](explore-dashboards.md) values when a user opens a dashboard, and available time zones and/or time ranges. + models: + refresh: + cron: '0 * * * *' + + metrics_views: + first_day_of_week: 1 + smallest_time_grain: month + + explores: + defaults: + time_range: P24M + time_zones: + - America/Denver + - UTC + - America/Los_Angeles + - America/Chicago + - America/New_York + - Europe/London + - Europe/Paris + - Asia/Jerusalem + - Europe/Moscow + - Asia/Kolkata + - Asia/Shanghai + - Asia/Tokyo + - Australia/Sydney + time_ranges: + - PT24H + - P7D + - P14D + - P30D + - P3M + - P6M + - P12M + - title: Setting variables + description: | + Primarily useful for [templating](/deploy/templating.md), variables can be set in the `rill.yaml` file directly. This allows variables to be set for your projects deployed to Rill Cloud while still being able to use different variable values locally if you prefer. + :::info Overriding variables locally + Variables also follow an order of precedence and can be overridden locally. By default, any variables defined will be inherited from `rill.yaml`. However, if you manually pass in a variable when starting Rill Developer locally via the CLI, this value will be used instead for the current instance of your running project: + ```bash + rill start --env numeric_var=100 --env string_var="different_value" + ``` + ::: + :::tip Setting variables through `.env` + Variables can also be set through your project's `/.env` file (or using the `rill env set` CLI command), such as: + ```bash + variable=xyz + ``` + Similar to how [connector credentials can be pushed / pulled](/build/credentials/credentials.md#pulling-credentials-and-variables-from-a-deployed-project-on-rill-cloud) from local to cloud or vice versa, project variables set locally in Rill Developer can be pushed to Rill Cloud and/or pulled back to your local instance from your deployed project by using the `rill env push` and `rill env pull` commands respectively. + ::: + type: object + properties: + env: + type: object + description: "To define a variable in `rill.yaml`, pass in the appropriate key-value pair for the variable under the `env` key" + examples: + - env: + numeric_var: 10 + string_var: "string_value" + - title: Managing Paths in Rill + description: | + The public_paths and ignore_paths properties in the rill.yaml file provide control over which files and directories are processed or exposed by Rill. The public_paths property defines a list of file or directory paths to expose over HTTP. By default, it includes ['./public']. The ignore_paths property specifies a list of files or directories that Rill excludes during ingestion and parsing. This prevents unnecessary or incompatible content from affecting the project. + :::tip + Don't forget the leading `/` when specifying the path for `ignore_paths` and this path is also assuming the relative path from your project root. + ::: + type: object + properties: + public_paths: + type: array + description: List of file or directory paths to expose over HTTP. Defaults to ['./public'] + items: + type: string + ignore_paths: + type: array + description: A list of file or directory paths to exclude from parsing. Useful for ignoring extraneous or non-Rill files in the project + examples: + - ignore_paths: + - /path/to/ignore + - /file_to_ignore.yaml + items: + type: string + - title: Testing access policies + description: | + During development, it is always a good idea to check if your [access policies](/manage/security.md) are behaving the way you designed them to before pushing these changes into production. You can set mock users which enables a drop down in the dashboard preview to view as a specific user. + :::info The View as selector is not visible in my dashboard, why? + This feature is _only_ enabled when you have set a security policy on the dashboard. By default, the dashboard and it's contents is viewable by every user. + ::: + type: object + properties: + mock_users: + type: array + description: A list of mock users used to test dashboard security policies within the project + examples: + - mock_users: + - email: john@yourcompany.com + name: John Doe + admin: true + - email: jane@partnercompany.com + groups: + - partners + - email: anon@unknown.com + - email: embed@rilldata.com + name: embed + custom_variable_1: Value_1 + custom_variable_2: Value_2 + + items: + type: object + properties: + email: + type: string + description: The email address of the mock user. This field is required + name: + type: string + description: The name of the mock user. + admin: + type: boolean + description: Indicates whether the mock user has administrative privileges + groups: + type: array + description: An array of group names that the mock user is a member of + items: + type: string + required: + - email + - $ref: '#/definitions/common_properties' definitions: common_properties: type: object @@ -54,32 +1468,39 @@ definitions: prod: type: object description: Overrides any properties in production environment. - component_variable_properties: - type: object - properties: - name: - type: string - description: Unique identifier for the variable - type: - type: string - description: Data type of the variable (e.g., string, number, boolean) - value: - description: Default value for the variable. Can be any valid JSON value type - type: - - string - - number - - boolean - - object - - array - required: - - name - - type - additionalProperties: false + # component_variable_properties: + # type: object + # properties: + # name: + # type: string + # description: Unique identifier for the variable + # type: + # type: string + # description: Data type of the variable (e.g., string, number, boolean) + # value: + # description: Default value for the variable. Can be any valid JSON value type + # type: + # - string + # - number + # - boolean + # - object + # - array + # required: + # - name + # - type + # additionalProperties: false data_properties: oneOf: - title: SQL Query type: object - description: Executes a raw SQL query against the project's data models. + description: | + Executes a raw SQL query against the project's data models. + ```yaml + type: api + + connector: "duckdb" + sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" + ``` properties: sql: type: string @@ -89,18 +1510,36 @@ definitions: description: specifies the connector to use when running SQL or glob queries. required: - sql + + - title: Metrics View Query type: object - description: Executes a SQL query that targets a defined metrics view. + description: | + Executes a SQL query that targets a defined metrics view. + ```yaml + type: api + + metrics_sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" + ``` properties: metrics_sql: type: string description: SQL query that targets a metrics view in the project required: - metrics_sql + - title: Custom API Call type: object - description: Calls a custom API defined in the project to compute data. + description: | + Calls a custom API defined in the project to compute data. + ```yaml + type: api + + api: "user_analytics_api" + args: + start_date: "2024-01-01" + limit: 10 + ``` properties: api: type: string @@ -111,15 +1550,23 @@ definitions: additionalProperties: true required: - api + - title: File Glob Query type: object - description: Uses a file-matching pattern (glob) to query data from a connector. + description: | + Uses a file-matching pattern (glob) to query data from a connector. + ```yaml + type: api + + glob: "data/*.csv" + ``` properties: glob: - description: Defines the file path or pattern to query from the specified connector. + description: Defines the file path or pattern to query from the specified connector. anyOf: - type: string description: A simple file path/glob pattern as a string. + sample: "data/*.csv" - type: object description: An object-based configuration for specifying a file path/glob pattern with advanced options. additionalProperties: true @@ -128,6 +1575,7 @@ definitions: description: Specifies the connector to use with the glob input. required: - glob + - title: Resource Status Check type: object description: Uses the status of a resource as data. @@ -139,9 +1587,15 @@ definitions: where_error: type: boolean description: Indicates whether the condition should trigger when the resource is in an error state. + sample: true additionalProperties: true required: - resource_status + examples: | + ```yaml + resource_status: + where_error: true + ``` explore_time_range_properties: oneOf: - type: string @@ -335,3 +1789,676 @@ definitions: description: SQL expression for row filtering (for row_filter type rules) required: - type + + dashboard_security_policy_properties: + type: object + description: Defines security rules and access control policies for dashboards (without row filtering) + properties: + access: + oneOf: + - type: string + description: SQL expression that evaluates to a boolean to determine access + - type: boolean + description: Direct boolean value to allow or deny access + description: Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + + + # Connector definitions + athena: + type: object + title: Athena + properties: + driver: + type: string + description: Refers to the driver type and must be driver `athena` + const: athena + aws_access_key_id: + type: string + description: AWS Access Key ID used for authentication. Required when using static credentials directly or as base credentials for assuming a role. + sample: "AKIAIOSFODNN7EXAMPLE" + aws_secret_access_key: + type: string + description: AWS Secret Access Key paired with the Access Key ID. Required when using static credentials directly or as base credentials for assuming a role. + sample: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + aws_access_token: + type: string + description: AWS session token used with temporary credentials. Required only if the Access Key and Secret Key are part of a temporary session credentials. + sample: "AKIAIOSFODNN7EXAMPLE" + role_arn: + type: string + description: ARN of the IAM role to assume. When specified, the SDK uses the base credentials to call STS AssumeRole and obtain temporary credentials scoped to this role. + sample: "arn:aws:iam::123456789012:role/MyRole" + role_session_name: + type: string + description: Session name to associate with the STS AssumeRole session. Used only if 'role_arn' is specified. Useful for identifying and auditing the session. + sample: "MySession" + external_id: + type: string + description: External ID required by some roles when assuming them, typically for cross-account access. Used only if 'role_arn' is specified and the role's trust policy requires it. + sample: "MyExternalID" + workgroup: + type: string + description: Athena workgroup to use for query execution. Defaults to 'primary' if not specified. + sample: "primary" + output_location: + type: string + description: S3 URI where Athena query results should be stored (e.g., s3://your-bucket/athena/results/). Optional if the selected workgroup has a default result configuration. + sample: "s3://my-bucket/athena-output/" + aws_region: + type: string + description: AWS region where Athena and the result S3 bucket are located (e.g., us-east-1). Defaults to 'us-east-1' if not specified. + sample: "us-east-1" + allow_host_access: + type: boolean + description: Allow the Athena client to access host environment configurations such as environment variables or local AWS credential files. Defaults to true, enabling use of credentials and settings from the host environment unless explicitly disabled. + sample: true + required: + - driver + azure: + type: object + title: Azure + properties: + driver: + type: string + description: Refers to the driver type and must be driver `azure` + const: azure + azure_storage_account: + type: string + description: Azure storage account name + sample: "mystorageaccount" + azure_storage_key: + type: string + description: Azure storage access key + sample: "myaccesskey" + azure_storage_bucket: + type: string + description: Name of the Azure Blob Storage container (equivalent to an S3 bucket) + sample: "my-container" + azure_storage_sas_token: + type: string + description: Optional azure SAS token for authentication + sample: "my-sas-token" + azure_storage_connection_string: + type: string + description: Optional azure connection string for storage account + sample: "DefaultEndpointsProtocol=https;AccountName=mystorageaccount;AccountKey=myaccesskey;EndpointSuffix=core.windows.net" + allow_host_access: + type: boolean + description: Allow access to host environment configuratio + sample: true + required: + - driver + - azure_storage_bucket + bigquery: + type: object + title: BigQuery + properties: + driver: + type: string + description: Refers to the driver type and must be driver `bigquery` + const: bigquery + google_application_credentials: + type: string + description: Raw contents of the Google Cloud service account key (in JSON format) used for authentication. + sample: "{\"type\": \"service_account\", \"project_id\": \"my-gcp-project\"}" + project_id: + type: string + description: Google Cloud project ID + sample: "my-gcp-project" + dataset_id: + type: string + description: BigQuery dataset ID + sample: "my_dataset" + location: + type: string + description: BigQuery dataset location + sample: "US" + allow_host_access: + type: boolean + description: Enable the BigQuery client to use credentials from the host environment when no service account JSON is provided. This includes Application Default Credentials from environment variables, local credential files, or the Google Compute Engine metadata server. Defaults to true, allowing seamless authentication in GCP environments. + sample: true + required: + - driver + clickhouse: + type: object + title: ClickHouse + properties: + driver: + type: string + description: Refers to the driver type and must be driver `clickhouse` + const: clickhouse + managed: + type: boolean + description: '`true` means Rill will provision the connector using the default provisioner. `false` disables automatic provisioning.' + sample: true + mode: + type: string + description: "`read` - Controls the operation mode for the ClickHouse connection. Defaults to 'read' for safe operation with external databases. Set to 'readwrite' to enable model creation and table mutations. Note: When 'managed: true', this is automatically set to 'readwrite'." + sample: "readwrite" + dsn: + type: string + description: DSN(Data Source Name) for the ClickHouse connection + sample: "clickhouse://localhost:9000/default" + username: + type: string + description: Username for authentication + sample: "default" + password: + type: string + description: Password for authentication + sample: "mypassword" + host: + type: string + description: Host where the ClickHouse instance is running + sample: "localhost" + port: + type: integer + description: Port where the ClickHouse instance is accessible + sample: 9000 + database: + type: string + description: Name of the ClickHouse database within the cluster + sample: "default" + ssl: + type: boolean + description: Indicates whether a secured SSL connection is required + sample: true + cluster: + type: string + description: 'Cluster name, required for running distributed queries' + sample: "my-cluster" + log_queries: + type: boolean + description: Controls whether to log raw SQL queries + sample: true + settings_override: + type: string + description: override the default settings used in queries. example `readonly = 1, session_timezone = 'UTC'` + sample: "readonly = 1, session_timezone = 'UTC'" + embed_port: + type: integer + description: Port to run ClickHouse locally (0 for random port) + sample: 0 + can_scale_to_zero: + type: boolean + description: Indicates if the database can scale to zero + sample: true + max_open_conns: + type: integer + description: Maximum number of open connections to the database + sample: 10 + max_idle_conns: + type: integer + description: Maximum number of idle connections in the pool + sample: 10 + dial_timeout: + type: string + description: Timeout for dialing the ClickHouse server + sample: "10s" + conn_max_lifetime: + type: string + description: Maximum time a connection may be reused + sample: "10s" + read_timeout: + type: string + description: Maximum time for a connection to read data + sample: "10s" + required: + - driver + druid: + type: object + title: Druid + properties: + driver: + type: string + description: Refers to the driver type and must be driver `druid` + const: druid + dsn: + type: string + description: Data Source Name (DSN) for connecting to Druid + sample: "http://localhost:8082" + username: + type: string + description: Username for authenticating with Druid + sample: "admin" + password: + type: string + description: Password for authenticating with Druid + sample: "admin123" + host: + type: string + description: Hostname of the Druid coordinator or broker + sample: "localhost" + port: + type: integer + description: Port number of the Druid service + sample: 8082 + ssl: + type: boolean + description: Enable SSL for secure connection + sample: true + log_queries: + type: boolean + description: Log raw SQL queries sent to Druid + sample: true + max_open_conns: + type: integer + description: Maximum number of open database connections (0 = default, -1 = unlimited) + sample: 10 + skip_version_check: + type: boolean + description: Skip checking Druid version compatibility + sample: true + required: + - driver + - dsn + duckdb: + type: object + title: DuckDB + properties: + driver: + type: string + description: Refers to the driver type and must be driver `duckdb` + const: duckdb + pool_size: + type: integer + description: Number of concurrent connections and queries allowed + sample: 10 + allow_host_access: + type: boolean + description: Whether access to the local environment and file system is allowed + sample: true + cpu: + type: integer + description: Number of CPU cores available to the database + sample: 10 + memory_limit_gb: + type: integer + description: Amount of memory in GB available to the database + sample: 10 + read_write_ratio: + type: number + description: Ratio of resources allocated to the read database; used to divide CPU and memory + sample: 0.5 + init_sql: + type: string + description: is executed during database initialization. + sample: "CREATE TABLE IF NOT EXISTS users (id INTEGER PRIMARY KEY, name TEXT, email TEXT)" + secrets: + type: string + description: Comma-separated list of other connector names to create temporary secrets for in DuckDB before executing a model. + sample: "gcs,s3" + log_queries: + type: boolean + description: Whether to log raw SQL queries executed through OLAP + sample: true + required: + - driver + gcs: + type: object + title: GCS + properties: + driver: + type: string + description: Refers to the driver type and must be driver `gcs` + const: gcs + google_application_credentials: + type: string + description: Google Cloud credentials JSON string + sample: "{\"type\": \"service_account\", \"project_id\": \"my-project\"}" + bucket: + type: string + description: Name of gcs bucket + sample: "my-gcs-bucket" + allow_host_access: + type: boolean + description: Allow access to host environment configuration + sample: true + key_id: + type: string + description: Optional S3-compatible Key ID when used in compatibility mode + sample: "AKIAIOSFODNN7EXAMPLE" + secret: + type: string + description: Optional S3-compatible Secret when used in compatibility mode + sample: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + required: + - driver + - bucket + https: + type: object + title: HTTPS + properties: + driver: + type: string + description: Refers to the driver type and must be driver `https` + const: https + path: + type: string + description: The full HTTPS URI to fetch data from + sample: "https://api.example.com/data.csv" + headers: + type: object + description: HTTP headers to include in the request + sample: "{\"Authorization\": \"Bearer my-token\"}" + additionalProperties: + type: string + required: + - driver + - path + motherduck: + type: object + title: MotherDuck + properties: + driver: + type: string + description: Refers to the driver type and must be driver `duckdb` + const: duckdb + path: + type: string + description: Path to your MD database + sample: "md:my_database" + init_sql: + type: string + description: SQL executed during database initialization. + sample: "INSTALL 'motherduck';\nLOAD 'motherduck';\nSET motherduck_token= '{{ .env.motherduck_token }}'" + required: + - driver + - path + - init_sql + mysql: + type: object + title: MySQL + properties: + driver: + type: string + description: Refers to the driver type and must be driver `mysql` + const: mysql + dsn: + type: string + description: DSN(Data Source Name) for the mysql connection + sample: "mysql://user:password@localhost:3306/mydatabase" + host: + type: string + description: Hostname of the MySQL server + sample: "localhost" + port: + type: integer + description: Port number for the MySQL server + sample: 3306 + database: + type: string + description: Name of the MySQL database + sample: "mydatabase" + user: + type: string + description: Username for authentication + sample: "myuser" + password: + type: string + description: Password for authentication + sample: "mypassword" + ssl_mode: + type: string + description: SSL mode can be DISABLED, PREFERRED or REQUIRED + sample: "PREFERRED" + required: + - driver + pinot: + type: object + title: Pinot + properties: + driver: + type: string + description: Refers to the driver type and must be driver `pinot` + const: pinot + dsn: + type: string + description: DSN(Data Source Name) for the Pinot connection + sample: "pinot://localhost:8099" + username: + type: string + description: Username for authenticating with Pinot + sample: "admin" + password: + type: string + description: Password for authenticating with Pinot + sample: "admin123" + broker_host: + type: string + description: Hostname of the Pinot broker + sample: "localhost" + broker_port: + type: integer + description: Port number for the Pinot broker + sample: 8099 + controller_host: + type: string + description: Hostname of the Pinot controller + sample: "localhost" + controller_port: + type: integer + description: Port number for the Pinot controller + sample: 9000 + ssl: + type: boolean + description: Enable SSL connection to Pinot + sample: true + log_queries: + type: boolean + description: Log raw SQL queries executed through Pinot + sample: true + max_open_conns: + type: integer + description: Maximum number of open connections to the Pinot database + sample: 10 + required: + - driver + - dsn + - broker_host + - controller_host + postgres: + type: object + title: Postgres + properties: + driver: + type: string + description: Refers to the driver type and must be driver `postgres` + const: postgres + dsn: + type: string + description: DSN(Data Source Name) for the postgres connection + sample: "postgresql://user:password@localhost:5432/mydatabase" + host: + type: string + description: Hostname of the Postgres server + sample: "localhost" + port: + type: string + description: Port number for the Postgres server + sample: "5432" + dbname: + type: string + description: Name of the Postgres database + sample: "mydatabase" + user: + type: string + description: Username for authentication + sample: "postgres" + password: + type: string + description: Password for authentication + sample: "mypassword" + sslmode: + type: string + description: SSL mode can be disable, allow, prefer or require + sample: "prefer" + required: + - driver + redshift: + type: object + title: Redshift + properties: + driver: + type: string + description: Refers to the driver type and must be driver `redshift` + const: redshift + aws_access_key_id: + type: string + description: AWS Access Key ID used for authenticating with Redshift. + sample: "AKIAIOSFODNN7EXAMPLE" + aws_secret_access_key: + type: string + description: AWS Secret Access Key used for authenticating with Redshift. + sample: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + aws_access_token: + type: string + description: AWS Session Token for temporary credentials (optional). + sample: "AKIAIOSFODNN7EXAMPLE" + region: + type: string + description: AWS region where the Redshift cluster or workgroup is hosted (e.g., 'us-east-1'). + sample: "us-east-1" + database: + type: string + description: Name of the Redshift database to query. + sample: "myredshiftdb" + workgroup: + type: string + description: Workgroup name for Redshift Serverless, in case of provisioned Redshift clusters use 'cluster_identifier'. + sample: "my-workgroup" + cluster_identifier: + type: string + description: Cluster identifier for provisioned Redshift clusters, in case of Redshift Serverless use 'workgroup' . + sample: "my-cluster" + required: + - driver + - aws_access_key_id + - aws_secret_access_key + - database + s3: + type: object + title: S3 + properties: + driver: + type: string + description: Refers to the driver type and must be driver `s3` + const: s3 + aws_access_key_id: + type: string + description: AWS Access Key ID used for authentication + sample: "AKIAIOSFODNN7EXAMPLE" + aws_secret_access_key: + type: string + description: AWS Secret Access Key used for authentication + sample: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + aws_access_token: + type: string + description: Optional AWS session token for temporary credentials + sample: "AKIAIOSFODNN7EXAMPLE" + bucket: + type: string + description: Name of s3 bucket + sample: "my-s3-bucket" + endpoint: + type: string + description: Optional custom endpoint URL for S3-compatible storage + sample: "https://s3.amazonaws.com" + region: + type: string + description: AWS region of the S3 bucket + sample: "us-east-1" + allow_host_access: + type: boolean + description: Allow access to host environment configuration + sample: true + retain_files: + type: boolean + description: Whether to retain intermediate files after processing + sample: true + required: + - driver + - bucket + salesforce: + type: object + title: Salesforce + properties: + driver: + type: string + description: Refers to the driver type and must be driver `salesforce` + const: salesforce + username: + type: string + description: Salesforce account username + sample: "user@example.com" + password: + type: string + description: Salesforce account password (secret) + sample: "mypassword" + key: + type: string + description: Authentication key for Salesforce (secret) + sample: "mysecretkey" + endpoint: + type: string + description: Salesforce API endpoint URL + sample: "https://login.salesforce.com" + client_id: + type: string + description: Client ID used for Salesforce OAuth authentication + sample: "myclientid" + required: + - driver + - username + - endpoint + slack: + type: object + title: Slack + properties: + driver: + type: string + description: Refers to the driver type and must be driver `slack` + const: slack + bot_token: + type: string + description: Bot token used for authenticating Slack API requests + sample: "xoxb-your-bot-token" + required: + - driver + - bot_token + snowflake: + type: object + title: Snowflake + properties: + driver: + type: string + description: Refers to the driver type and must be driver `snowflake` + const: snowflake + dsn: + type: string + description: DSN (Data Source Name) for the Snowflake connection + sample: "user:password@account/database/schema?warehouse=warehouse" + parallel_fetch_limit: + type: integer + description: Maximum number of concurrent fetches during query execution + sample: 10 + required: + - driver + - dsn + sqlite: + type: object + title: SQLite + properties: + driver: + type: string + description: Refers to the driver type and must be driver `sqlite` + const: sqlite + dsn: + type: string + description: DSN(Data Source Name) for the sqlite connection + sample: "file:./mydatabase.db" + required: + - driver + - dsn + - driver + - dsn \ No newline at end of file From 98baf5623514de6250662d60b043466af9913013 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Wed, 6 Aug 2025 15:39:25 -0600 Subject: [PATCH 21/32] Update generate_project.go --- cli/cmd/docs/generate_project.go | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index 916adbe1b0b..818928af861 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -423,24 +423,37 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req for _, example := range examples.Content { b, err := yaml.Marshal(example) if err != nil { + panic(err) + } + doc.WriteString(fmt.Sprintf("\n\n```yaml\n%s```", string(b))) } - doc.WriteString(fmt.Sprintf("\n\n```yaml\n%s```", string(b))) + } else if examples.Kind == yaml.ScalarNode { + // Handle string examples (like markdown code blocks) + doc.WriteString(fmt.Sprintf("\n\n%s", examples.Value)) } } } } else if items := getNodeForKey(node, "items"); items != nil && items.Kind == yaml.MappingNode { items := getNodeForKey(node, "items") - doc.WriteString(generateDoc(sidebarPosition, level, items, indent, getRequiredMapFromNode(items))) + doc.WriteString(generateDoc(sidebarPosition, level, items, indent, getRequiredMapFromNode(items), rootSchema, id)) } // OneOf if oneOf := getNodeForKey(node, "oneOf"); oneOf != nil && oneOf.Kind == yaml.SequenceNode { - // Skip oneOf processing for connectors at level 1 since we handle it in allOf - if !(isConnector && level == 1) { - if len(oneOf.Content) == 1 { - doc.WriteString(generateDoc(sidebarPosition, level, oneOf.Content[0], indent, getRequiredMapFromNode(oneOf.Content[0]))) - if level == 1 { - doc.WriteString("\n\n## One of Properties Options") + // Special handling for connectors - generate copiable text for each connector type + if id == "connectors" { + doc.WriteString("\n\n## Available Connector Types\n\n") + for _, item := range oneOf.Content { + // Since $ref values are already resolved, look for connector definitions directly + title := getScalarValue(item, "title") + if title != "" { + doc.WriteString(fmt.Sprintf("\n\n### %s\n\n", title)) + // Add description first + description := getPrintableDescription(item, indent, "") + if description != "" { + doc.WriteString(fmt.Sprintf("%s\n\n", description)) + } + } // Generate the connector definition documentation (properties, etc.) if properties := getNodeForKey(item, "properties"); properties != nil && properties.Kind == yaml.MappingNode { From 7065bcc181a62931f77671f621d0bc4b0eb02dba Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:03:44 -0600 Subject: [PATCH 22/32] good old cursor wiped my functoni --- cli/cmd/docs/generate_project.go | 88 +++++++++ docs/docs/hidden/yaml/connectors.md | 278 ++++++++++++++++++++++++++++ 2 files changed, 366 insertions(+) diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index 818928af861..79a05f5c41e 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -446,8 +446,12 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req for _, item := range oneOf.Content { // Since $ref values are already resolved, look for connector definitions directly title := getScalarValue(item, "title") + exampleOutput := generateConnectorExample(title, item) + if title != "" { doc.WriteString(fmt.Sprintf("\n\n### %s\n\n", title)) + // Generate copiable example for this connector + doc.WriteString(fmt.Sprintf("\n\n\n%s", exampleOutput)) // Add description first description := getPrintableDescription(item, indent, "") if description != "" { @@ -472,6 +476,8 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req required)) } } + + } } else { @@ -561,6 +567,8 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req doc.WriteString(fmt.Sprintf("%s\n\n", description)) } } + exampleOutput := generateConnectorExample(title, connectorDef) + doc.WriteString(fmt.Sprintf("\n\n#### Example\n\n%s", exampleOutput)) // Generate the connector definition documentation (properties, etc.) but skip the header // We need to process properties manually to avoid duplicate headers @@ -580,6 +588,7 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req required)) } } + } } @@ -620,3 +629,82 @@ func hasCombinators(node *yaml.Node) bool { return getNodeForKey(node, "anyOf") != nil || getNodeForKey(node, "oneOf") != nil || getNodeForKey(node, "allOf") != nil } +func generateConnectorExample(connectorType string, connectorDef *yaml.Node) string { + if connectorDef == nil { + return "" + } + + var example strings.Builder + example.WriteString("```yaml\n") + example.WriteString("type: connector # Must be `connector` (required)\n") + + // Get the driver from the schema and add it first + driverAdded := false + if driver := getNodeForKey(connectorDef, "driver"); driver != nil { + if constVal := getScalarValue(driver, "const"); constVal != "" { + example.WriteString(fmt.Sprintf("driver: %s # Must be `%s` _(required)_\n\n", constVal, constVal)) + driverAdded = true + } + } + + // Fallback: if driver wasn't found, use the connector type name + if !driverAdded { + // Special case for MotherDuck which uses duckdb driver + if connectorType == "MotherDuck" { + example.WriteString("driver: duckdb # Must be `duckdb` _(required)_\n\n") + } else { + example.WriteString(fmt.Sprintf("driver: %s # Must be `%s` _(required)_\n\n", strings.ToLower(connectorType), strings.ToLower(connectorType))) + } + } + + // Get all properties from the schema + if properties := getNodeForKey(connectorDef, "properties"); properties != nil && properties.Kind == yaml.MappingNode { + for i := 0; i < len(properties.Content); i += 2 { + propName := properties.Content[i].Value + propValue := properties.Content[i+1] + + // Skip the driver property since we already added it + if propName == "driver" { + continue + } + + // Get property description + description := getPrintableDescription(propValue, "", "") + if description == "" { + description = "Property description" + } + + // Get sample value from the schema + sampleValue := getScalarValue(propValue, "sample") + if sampleValue == "" { + // Fallback to const value if no sample + sampleValue = getScalarValue(propValue, "const") + } + if sampleValue == "" { + // Final fallback + sampleValue = "example_value" + } + + // Check if it's required + required := "" + if requiredFields := getRequiredMapFromNode(connectorDef); requiredFields[propName] { + required = " _(required)_" + } + + // Format the line with proper alignment + example.WriteString(fmt.Sprintf("%s: %s", propName, sampleValue)) + + // Add padding for alignment + padding := 35 - len(propName) - len(sampleValue) + if padding > 0 { + example.WriteString(strings.Repeat(" ", padding)) + } + + example.WriteString(fmt.Sprintf("# %s%s\n", description, required)) + } + } + + example.WriteString("```\n\n") + return example.String() +} + diff --git a/docs/docs/hidden/yaml/connectors.md b/docs/docs/hidden/yaml/connectors.md index 204da86883e..d5800bfade1 100644 --- a/docs/docs/hidden/yaml/connectors.md +++ b/docs/docs/hidden/yaml/connectors.md @@ -73,6 +73,25 @@ _[object]_ - Overrides any properties in production environment. + +```yaml +type: connector # Must be `connector` (required) +driver: athena # Must be `athena` _(required)_ + +aws_access_key_id: AKIAIOSFODNN7EXAMPLE# AWS Access Key ID used for authentication. Required when using static credentials directly or as base credentials for assuming a role. +aws_secret_access_key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY# AWS Secret Access Key paired with the Access Key ID. Required when using static credentials directly or as base credentials for assuming a role. +aws_access_token: AKIAIOSFODNN7EXAMPLE# AWS session token used with temporary credentials. Required only if the Access Key and Secret Key are part of a temporary session credentials. +role_arn: arn:aws:iam::123456789012:role/MyRole# ARN of the IAM role to assume. When specified, the SDK uses the base credentials to call STS AssumeRole and obtain temporary credentials scoped to this role. +role_session_name: MySession # Session name to associate with the STS AssumeRole session. Used only if 'role_arn' is specified. Useful for identifying and auditing the session. +external_id: MyExternalID # External ID required by some roles when assuming them, typically for cross-account access. Used only if 'role_arn' is specified and the role's trust policy requires it. +workgroup: primary # Athena workgroup to use for query execution. Defaults to 'primary' if not specified. +output_location: s3://my-bucket/athena-output/# S3 URI where Athena query results should be stored (e.g., s3://your-bucket/athena/results/). Optional if the selected workgroup has a default result configuration. +aws_region: us-east-1 # AWS region where Athena and the result S3 bucket are located (e.g., us-east-1). Defaults to 'us-east-1' if not specified. +allow_host_access: true # Allow the Athena client to access host environment configurations such as environment variables or local AWS credential files. Defaults to true, enabling use of credentials and settings from the host environment unless explicitly disabled. +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `athena` _(required)_ @@ -121,6 +140,21 @@ _[boolean]_ - Allow the Athena client to access host environment configurations + +```yaml +type: connector # Must be `connector` (required) +driver: azure # Must be `azure` _(required)_ + +azure_storage_account: mystorageaccount# Azure storage account name +azure_storage_key: myaccesskey # Azure storage access key +azure_storage_bucket: my-container # Name of the Azure Blob Storage container (equivalent to an S3 bucket) _(required)_ +azure_storage_sas_token: my-sas-token# Optional azure SAS token for authentication +azure_storage_connection_string: DefaultEndpointsProtocol=https;AccountName=mystorageaccount;AccountKey=myaccesskey;EndpointSuffix=core.windows.net# Optional azure connection string for storage account +allow_host_access: true # Allow access to host environment configuratio +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `azure` _(required)_ @@ -153,6 +187,20 @@ _[boolean]_ - Allow access to host environment configuratio + +```yaml +type: connector # Must be `connector` (required) +driver: bigquery # Must be `bigquery` _(required)_ + +google_application_credentials: {"type": "service_account", "project_id": "my-gcp-project"}# Raw contents of the Google Cloud service account key (in JSON format) used for authentication. +project_id: my-gcp-project # Google Cloud project ID +dataset_id: my_dataset # BigQuery dataset ID +location: US # BigQuery dataset location +allow_host_access: true # Enable the BigQuery client to use credentials from the host environment when no service account JSON is provided. This includes Application Default Credentials from environment variables, local credential files, or the Google Compute Engine metadata server. Defaults to true, allowing seamless authentication in GCP environments. +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `bigquery` _(required)_ @@ -181,6 +229,34 @@ _[boolean]_ - Enable the BigQuery client to use credentials from the host enviro + +```yaml +type: connector # Must be `connector` (required) +driver: clickhouse # Must be `clickhouse` _(required)_ + +managed: true # `true` means Rill will provision the connector using the default provisioner. `false` disables automatic provisioning. +mode: readwrite # `read` - Controls the operation mode for the ClickHouse connection. Defaults to 'read' for safe operation with external databases. Set to 'readwrite' to enable model creation and table mutations. Note: When 'managed: true', this is automatically set to 'readwrite'. +dsn: clickhouse://localhost:9000/default# DSN(Data Source Name) for the ClickHouse connection +username: default # Username for authentication +password: mypassword # Password for authentication +host: localhost # Host where the ClickHouse instance is running +port: 9000 # Port where the ClickHouse instance is accessible +database: default # Name of the ClickHouse database within the cluster +ssl: true # Indicates whether a secured SSL connection is required +cluster: my-cluster # Cluster name, required for running distributed queries +log_queries: true # Controls whether to log raw SQL queries +settings_override: readonly = 1, session_timezone = 'UTC'# override the default settings used in queries. example `readonly = 1, session_timezone = 'UTC'` +embed_port: 0 # Port to run ClickHouse locally (0 for random port) +can_scale_to_zero: true # Indicates if the database can scale to zero +max_open_conns: 10 # Maximum number of open connections to the database +max_idle_conns: 10 # Maximum number of idle connections in the pool +dial_timeout: 10s # Timeout for dialing the ClickHouse server +conn_max_lifetime: 10s # Maximum time a connection may be reused +read_timeout: 10s # Maximum time for a connection to read data +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `clickhouse` _(required)_ @@ -265,6 +341,24 @@ _[string]_ - Maximum time for a connection to read data + +```yaml +type: connector # Must be `connector` (required) +driver: druid # Must be `druid` _(required)_ + +dsn: http://localhost:8082 # Data Source Name (DSN) for connecting to Druid _(required)_ +username: admin # Username for authenticating with Druid +password: admin123 # Password for authenticating with Druid +host: localhost # Hostname of the Druid coordinator or broker +port: 8082 # Port number of the Druid service +ssl: true # Enable SSL for secure connection +log_queries: true # Log raw SQL queries sent to Druid +max_open_conns: 10 # Maximum number of open database connections (0 = default, -1 = unlimited) +skip_version_check: true # Skip checking Druid version compatibility +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `druid` _(required)_ @@ -309,6 +403,23 @@ _[boolean]_ - Skip checking Druid version compatibility + +```yaml +type: connector # Must be `connector` (required) +driver: duckdb # Must be `duckdb` _(required)_ + +pool_size: 10 # Number of concurrent connections and queries allowed +allow_host_access: true # Whether access to the local environment and file system is allowed +cpu: 10 # Number of CPU cores available to the database +memory_limit_gb: 10 # Amount of memory in GB available to the database +read_write_ratio: 0.5 # Ratio of resources allocated to the read database; used to divide CPU and memory +init_sql: CREATE TABLE IF NOT EXISTS users (id INTEGER PRIMARY KEY, name TEXT, email TEXT)# is executed during database initialization. +secrets: gcs,s3 # Comma-separated list of other connector names to create temporary secrets for in DuckDB before executing a model. +log_queries: true # Whether to log raw SQL queries executed through OLAP +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `duckdb` _(required)_ @@ -349,6 +460,20 @@ _[boolean]_ - Whether to log raw SQL queries executed through OLAP + +```yaml +type: connector # Must be `connector` (required) +driver: gcs # Must be `gcs` _(required)_ + +google_application_credentials: {"type": "service_account", "project_id": "my-project"}# Google Cloud credentials JSON string +bucket: my-gcs-bucket # Name of gcs bucket _(required)_ +allow_host_access: true # Allow access to host environment configuration +key_id: AKIAIOSFODNN7EXAMPLE # Optional S3-compatible Key ID when used in compatibility mode +secret: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY# Optional S3-compatible Secret when used in compatibility mode +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `gcs` _(required)_ @@ -377,6 +502,17 @@ _[string]_ - Optional S3-compatible Secret when used in compatibility mode + +```yaml +type: connector # Must be `connector` (required) +driver: https # Must be `https` _(required)_ + +path: https://api.example.com/data.csv# The full HTTPS URI to fetch data from _(required)_ +headers: {"Authorization": "Bearer my-token"}# HTTP headers to include in the request +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `https` _(required)_ @@ -393,6 +529,19 @@ _[object]_ - HTTP headers to include in the request + +```yaml +type: connector # Must be `connector` (required) +driver: duckdb # Must be `duckdb` _(required)_ + +path: md:my_database # Path to your MD database _(required)_ +init_sql: INSTALL 'motherduck'; +LOAD 'motherduck'; +SET motherduck_token= '{{ .env.motherduck_token }}'# SQL executed during database initialization. _(required)_ +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `duckdb` _(required)_ @@ -409,6 +558,22 @@ _[string]_ - SQL executed during database initialization. _(required)_ + +```yaml +type: connector # Must be `connector` (required) +driver: mysql # Must be `mysql` _(required)_ + +dsn: mysql://user:password@localhost:3306/mydatabase# DSN(Data Source Name) for the mysql connection +host: localhost # Hostname of the MySQL server +port: 3306 # Port number for the MySQL server +database: mydatabase # Name of the MySQL database +user: myuser # Username for authentication +password: mypassword # Password for authentication +ssl_mode: PREFERRED # SSL mode can be DISABLED, PREFERRED or REQUIRED +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `mysql` _(required)_ @@ -445,6 +610,25 @@ _[string]_ - SSL mode can be DISABLED, PREFERRED or REQUIRED + +```yaml +type: connector # Must be `connector` (required) +driver: pinot # Must be `pinot` _(required)_ + +dsn: pinot://localhost:8099 # DSN(Data Source Name) for the Pinot connection _(required)_ +username: admin # Username for authenticating with Pinot +password: admin123 # Password for authenticating with Pinot +broker_host: localhost # Hostname of the Pinot broker _(required)_ +broker_port: 8099 # Port number for the Pinot broker +controller_host: localhost # Hostname of the Pinot controller _(required)_ +controller_port: 9000 # Port number for the Pinot controller +ssl: true # Enable SSL connection to Pinot +log_queries: true # Log raw SQL queries executed through Pinot +max_open_conns: 10 # Maximum number of open connections to the Pinot database +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `pinot` _(required)_ @@ -493,6 +677,22 @@ _[integer]_ - Maximum number of open connections to the Pinot database + +```yaml +type: connector # Must be `connector` (required) +driver: postgres # Must be `postgres` _(required)_ + +dsn: postgresql://user:password@localhost:5432/mydatabase# DSN(Data Source Name) for the postgres connection +host: localhost # Hostname of the Postgres server +port: 5432 # Port number for the Postgres server +dbname: mydatabase # Name of the Postgres database +user: postgres # Username for authentication +password: mypassword # Password for authentication +sslmode: prefer # SSL mode can be disable, allow, prefer or require +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `postgres` _(required)_ @@ -529,6 +729,22 @@ _[string]_ - SSL mode can be disable, allow, prefer or require + +```yaml +type: connector # Must be `connector` (required) +driver: redshift # Must be `redshift` _(required)_ + +aws_access_key_id: AKIAIOSFODNN7EXAMPLE# AWS Access Key ID used for authenticating with Redshift. _(required)_ +aws_secret_access_key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY# AWS Secret Access Key used for authenticating with Redshift. _(required)_ +aws_access_token: AKIAIOSFODNN7EXAMPLE# AWS Session Token for temporary credentials (optional). +region: us-east-1 # AWS region where the Redshift cluster or workgroup is hosted (e.g., 'us-east-1'). +database: myredshiftdb # Name of the Redshift database to query. _(required)_ +workgroup: my-workgroup # Workgroup name for Redshift Serverless, in case of provisioned Redshift clusters use 'cluster_identifier'. +cluster_identifier: my-cluster # Cluster identifier for provisioned Redshift clusters, in case of Redshift Serverless use 'workgroup' . +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `redshift` _(required)_ @@ -565,6 +781,23 @@ _[string]_ - Cluster identifier for provisioned Redshift clusters, in case of Re + +```yaml +type: connector # Must be `connector` (required) +driver: s3 # Must be `s3` _(required)_ + +aws_access_key_id: AKIAIOSFODNN7EXAMPLE# AWS Access Key ID used for authentication +aws_secret_access_key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY# AWS Secret Access Key used for authentication +aws_access_token: AKIAIOSFODNN7EXAMPLE# Optional AWS session token for temporary credentials +bucket: my-s3-bucket # Name of s3 bucket _(required)_ +endpoint: https://s3.amazonaws.com # Optional custom endpoint URL for S3-compatible storage +region: us-east-1 # AWS region of the S3 bucket +allow_host_access: true # Allow access to host environment configuration +retain_files: true # Whether to retain intermediate files after processing +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `s3` _(required)_ @@ -605,6 +838,20 @@ _[boolean]_ - Whether to retain intermediate files after processing + +```yaml +type: connector # Must be `connector` (required) +driver: salesforce # Must be `salesforce` _(required)_ + +username: user@example.com # Salesforce account username _(required)_ +password: mypassword # Salesforce account password (secret) +key: mysecretkey # Authentication key for Salesforce (secret) +endpoint: https://login.salesforce.com# Salesforce API endpoint URL _(required)_ +client_id: myclientid # Client ID used for Salesforce OAuth authentication +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `salesforce` _(required)_ @@ -633,6 +880,16 @@ _[string]_ - Client ID used for Salesforce OAuth authentication + +```yaml +type: connector # Must be `connector` (required) +driver: slack # Must be `slack` _(required)_ + +bot_token: xoxb-your-bot-token # Bot token used for authenticating Slack API requests _(required)_ +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `slack` _(required)_ @@ -645,6 +902,17 @@ _[string]_ - Bot token used for authenticating Slack API requests _(required)_ + +```yaml +type: connector # Must be `connector` (required) +driver: snowflake # Must be `snowflake` _(required)_ + +dsn: user:password@account/database/schema?warehouse=warehouse# DSN (Data Source Name) for the Snowflake connection _(required)_ +parallel_fetch_limit: 10 # Maximum number of concurrent fetches during query execution +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `snowflake` _(required)_ @@ -661,6 +929,16 @@ _[integer]_ - Maximum number of concurrent fetches during query execution + +```yaml +type: connector # Must be `connector` (required) +driver: sqlite # Must be `sqlite` _(required)_ + +dsn: file:./mydatabase.db # DSN(Data Source Name) for the sqlite connection _(required)_ +``` + + + #### `driver` _[string]_ - Refers to the driver type and must be driver `sqlite` _(required)_ From dba395a9fbda2598212de5fa339fb1070e965931 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:07:21 -0600 Subject: [PATCH 23/32] missing parameters --- docs/docs/hidden/yaml/alerts.md | 58 ++++++++++++- runtime/parser/schema/project.schema.yaml | 100 +++++++++++++++++++++- 2 files changed, 153 insertions(+), 5 deletions(-) diff --git a/docs/docs/hidden/yaml/alerts.md b/docs/docs/hidden/yaml/alerts.md index 79e9cf23ca1..7ab57a3b326 100644 --- a/docs/docs/hidden/yaml/alerts.md +++ b/docs/docs/hidden/yaml/alerts.md @@ -20,7 +20,7 @@ _[object]_ - Refresh schedule for the alert cron: "* * * * *" #every: "24h" ``` - + _(required)_ - **`cron`** - _[string]_ - A cron expression that defines the execution schedule @@ -34,7 +34,7 @@ _[object]_ - Refresh schedule for the alert ### `display_name` -_[string]_ - Display name for the alert _(required)_ +_[string]_ - Display name for the alert ### `description` @@ -72,9 +72,57 @@ resource_status: ``` +### `intervals` + +_[object]_ - define the interval of the alert to check + + - **`duration`** - _[string]_ - a valid ISO8601 duration to define the interval duration + + - **`limit`** - _[integer]_ - maximum number of intervals to check for on invocation + + - **`check_unclosed`** - _[boolean]_ - boolean, whether unclosed intervals should be checked + +### `watermark` + +_[string]_ - Specifies how the watermark is determined for incremental processing. Use 'trigger_time' to set it at runtime or 'inherit' to use the upstream model's watermark. + +### `timeout` + +_[string]_ - define the timeout of the alert in seconds (optional). + +### `for` + +_[oneOf]_ - Specifies how user identity or attributes should be evaluated for security policy enforcement. + + - **`user_id`** - _[string]_ - The unique user ID used to evaluate security policies. _(required)_ + + - **`user_email`** - _[string]_ - The user's email address used to evaluate security policies. _(required)_ + + - **`attributes`** - _[object]_ - A dictionary of user attributes used to evaluate security policies. _(required)_ + +### `on_recover` + +_[boolean]_ - Send an alert when a previously failing alert recovers. Defaults to false. + +### `on_fail` + +_[boolean]_ - Send an alert when a failure occurs. Defaults to true. + +### `on_error` + +_[boolean]_ - Send an alert when an error occurs during evaluation. Defaults to false. + +### `renotify` + +_[boolean]_ - Enable repeated notifications for unresolved alerts. Defaults to false. + +### `renotify_after` + +_[string]_ - Defines the re-notification interval for the alert (e.g., '10m','24h'), equivalent to snooze duration in UI, defaults to 'Off' + ### `condition` -_[object]_ - Condition that triggers the alert _(required)_ +_[object]_ - Condition that triggers the alert - **`operator`** - _[string]_ - Comparison operator (gt, lt, eq, etc.) @@ -98,6 +146,10 @@ _[object]_ - Notification configuration _(required)_ - **`webhooks`** - _[array of string]_ - An array of Slack webhook URLs to send notifications to. +### `annotations` + +_[object]_ - Key value pair used for annotations + ## Common Properties ### `name` diff --git a/runtime/parser/schema/project.schema.yaml b/runtime/parser/schema/project.schema.yaml index 8fbae53a33e..56c25913c15 100644 --- a/runtime/parser/schema/project.schema.yaml +++ b/runtime/parser/schema/project.schema.yaml @@ -1100,6 +1100,28 @@ oneOf: id: alerts type: object description: Along with alertings at the dashboard level and can be created via the UI, there might be more extensive alerting that you might want to develop and can be done so the an alert.yaml. When creating an alert via a YAML file, you'll see this denoted in the UI as `Created through code`. + examples: + - # Example: To send alert when data lags by more than 1 day to slack channel #rill-cloud-alerts + type: alert + display_name: Data lags by more than 1 day + # Check the alert every hour. + refresh: + cron: 0 * * * * + # Query that returns non-empty results if the metrics lag by more than 1 day. + data: + sql: |- + SELECT * + FROM + ( + SELECT MAX(event_time) AS max_time + FROM rill_metrics_model + ) + WHERE max_time < NOW() - INTERVAL '1 day' + # Send notifications in Slack. + notify: + slack: + channels: + - '#rill-cloud-alerts' allOf: - title: Properties type: object @@ -1126,6 +1148,76 @@ oneOf: data: $ref: '#/definitions/data_properties' description: Data source for the alert + intervals: + type: object + description: define the interval of the alert to check + properties: + duration: + type: string + description: a valid ISO8601 duration to define the interval duration + limit: + type: integer + description: maximum number of intervals to check for on invocation + minimum: 0 + check_unclosed: + type: boolean + description: 'boolean, whether unclosed intervals should be checked' + watermark: + type: string + enum: + - trigger_time + - inherit + description: Specifies how the watermark is determined for incremental processing. Use 'trigger_time' to set it at runtime or 'inherit' to use the upstream model's watermark. + timeout: + type: string + description: define the timeout of the alert in seconds (optional). + for: + description: "Specifies how user identity or attributes should be evaluated for security policy enforcement." + oneOf: + - type: object + description: Specifies a unique user identifier for applying security policies. + properties: + user_id: + type: string + description: "The unique user ID used to evaluate security policies." + required: + - user_id + additionalProperties: false + - type: object + description: Specifies a user's email address for applying security policies. + properties: + user_email: + type: string + description: "The user's email address used to evaluate security policies." + format: email + required: + - user_email + additionalProperties: false + - type: object + description: Specifies a set of arbitrary user attributes for applying security policies. + properties: + attributes: + type: object + description: A dictionary of user attributes used to evaluate security policies. + additionalProperties: true + required: + - attributes + additionalProperties: false + on_recover: + type: boolean + description: Send an alert when a previously failing alert recovers. Defaults to false. + on_fail: + type: boolean + description: Send an alert when a failure occurs. Defaults to true. + on_error: + type: boolean + description: Send an alert when an error occurs during evaluation. Defaults to false. + renotify: + type: boolean + description: Enable repeated notifications for unresolved alerts. Defaults to false. + renotify_after: + type: string + description: Defines the re-notification interval for the alert (e.g., '10m','24h'), equivalent to snooze duration in UI, defaults to 'Off' condition: type: object description: Condition that triggers the alert @@ -1141,12 +1233,16 @@ oneOf: notify: $ref: '#/definitions/notify_properties' description: Notification configuration + annotations: + type: object + description: Key value pair used for annotations + additionalProperties: + type: string required: - type - - display_name + - refresh - data - - condition - notify - $ref: '#/definitions/common_properties' From 6e32ddd9eddffcb1d6316d25e51da405cb0c1b66 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:18:50 -0600 Subject: [PATCH 24/32] midding parameters and inline examples --- docs/docs/hidden/yaml/advanced-models.md | 47 +++++++++++++++++++++-- docs/docs/hidden/yaml/alerts.md | 28 +++++++++++++- runtime/parser/schema/project.schema.yaml | 47 +++++++++++++++++++++-- 3 files changed, 115 insertions(+), 7 deletions(-) diff --git a/docs/docs/hidden/yaml/advanced-models.md b/docs/docs/hidden/yaml/advanced-models.md index 253513acaa2..3d717bad67f 100644 --- a/docs/docs/hidden/yaml/advanced-models.md +++ b/docs/docs/hidden/yaml/advanced-models.md @@ -54,6 +54,24 @@ _[string]_ - Refers to the resource type and is needed if setting an explicit OL _[string]_ - Raw SQL query to run against source _(required)_ +### `pre_exec` + +_[string]_ - Refers to SQL queries to run before the main query, available for DuckDB-based models. (optional). +Ensure pre_exec queries are idempotent. Use IF NOT EXISTS statements when applicable. +```yaml +pre_exec: ATTACH IF NOT EXISTS 'dbname=postgres host=localhost port=5432 user=postgres password=postgres' AS postgres_db (TYPE POSTGRES) +``` + + +### `post_exec` + +_[string]_ - Refers to a SQL query that is run after the main query, available for DuckDB-based models. (optional). +Ensure post_exec queries are idempotent. Use IF EXISTS statements when applicable. +```yaml +post_exec: DETACH DATABASE IF EXISTS postgres_db +``` + + ### `timeout` _[string]_ - The maximum time to wait for model ingestion @@ -68,7 +86,12 @@ _[string]_ - Configure how changes to the model specifications are applied (opti ### `state` -_[oneOf]_ - Refers to the explicitly defined state of your model, cannot be used with partitions (optional) +_[oneOf]_ - Refers to the explicitly defined state of your model, cannot be used with partitions (optional) +```yaml +state: + sql: SELECT MAX(date) as max_date +``` + - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ @@ -100,7 +123,17 @@ resource_status: ### `partitions` -_[oneOf]_ - Refers to the how your data is partitioned, cannot be used with state. (optional) +_[oneOf]_ - Refers to the how your data is partitioned, cannot be used with state. (optional) +```yaml +partitions: + glob: gcs://my_bucket/y=*/m=*/d=*/*.parquet +``` +```yaml +partitions: + connector: duckdb + sql: SELECT range AS num FROM range(0,10) + ``` + - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ @@ -144,10 +177,18 @@ _[integer]_ - Refers to the number of concurrent partitions that can be read at ### `stage` -_[object]_ - in the case of staging models, where an input source does not support direct write to the output and a staging table is required +_[object]_ - in the case of staging models, where an input source does not support direct write to the output and a staging table is required +```yaml +stage: + connector: s3 + path: s3://my_bucket/my_staging_table +``` + - **`connector`** - _[string]_ - Refers to the connector type for the staging table _(required)_ + - **`path`** - _[string]_ - Refers to the path to the staging table + ### `output` _[object]_ - to define the properties of output diff --git a/docs/docs/hidden/yaml/alerts.md b/docs/docs/hidden/yaml/alerts.md index 7ab57a3b326..2643a3503e2 100644 --- a/docs/docs/hidden/yaml/alerts.md +++ b/docs/docs/hidden/yaml/alerts.md @@ -166,4 +166,30 @@ _[object]_ - Overrides any properties in development environment. ### `prod` -_[object]_ - Overrides any properties in production environment. \ No newline at end of file +_[object]_ - Overrides any properties in production environment. + +## Examples + +```yaml +# Example: To send alert when data lags by more than 1 day to slack channel #rill-cloud-alerts +type: alert +display_name: Data lags by more than 1 day +# Check the alert every hour. +refresh: + cron: 0 * * * * +# Query that returns non-empty results if the metrics lag by more than 1 day. +data: + sql: |- + SELECT * + FROM + ( + SELECT MAX(event_time) AS max_time + FROM rill_metrics_model + ) + WHERE max_time < NOW() - INTERVAL '1 day' +# Send notifications in Slack. +notify: + slack: + channels: + - '#rill-cloud-alerts' +``` \ No newline at end of file diff --git a/runtime/parser/schema/project.schema.yaml b/runtime/parser/schema/project.schema.yaml index 56c25913c15..b75426f188a 100644 --- a/runtime/parser/schema/project.schema.yaml +++ b/runtime/parser/schema/project.schema.yaml @@ -293,6 +293,22 @@ oneOf: sql: type: string description: Raw SQL query to run against source + pre_exec: + type: string + description: | + Refers to SQL queries to run before the main query, available for DuckDB-based models. (optional). + Ensure pre_exec queries are idempotent. Use IF NOT EXISTS statements when applicable. + ```yaml + pre_exec: ATTACH IF NOT EXISTS 'dbname=postgres host=localhost port=5432 user=postgres password=postgres' AS postgres_db (TYPE POSTGRES) + ``` + post_exec: + type: string + description: | + Refers to a SQL query that is run after the main query, available for DuckDB-based models. (optional). + Ensure post_exec queries are idempotent. Use IF EXISTS statements when applicable. + ```yaml + post_exec: DETACH DATABASE IF EXISTS postgres_db + ``` timeout: type: string description: The maximum time to wait for model ingestion @@ -308,10 +324,25 @@ oneOf: description: Configure how changes to the model specifications are applied (optional). 'reset' will drop and recreate the model automatically, 'manual' will require a manual full or incremental refresh to apply changes, and 'patch' will switch to the new logic without re-processing historical data (only applies for incremental models). state: $ref: '#/definitions/data_properties' - description: Refers to the explicitly defined state of your model, cannot be used with partitions (optional) + description: | + Refers to the explicitly defined state of your model, cannot be used with partitions (optional) + ```yaml + state: + sql: SELECT MAX(date) as max_date + ``` partitions: $ref: '#/definitions/data_properties' - description: Refers to the how your data is partitioned, cannot be used with state. (optional) + description: | + Refers to the how your data is partitioned, cannot be used with state. (optional) + ```yaml + partitions: + glob: gcs://my_bucket/y=*/m=*/d=*/*.parquet + ``` + ```yaml + partitions: + connector: duckdb + sql: SELECT range AS num FROM range(0,10) + ``` materialize: type: boolean description: models will be materialized in olap @@ -327,9 +358,19 @@ oneOf: connector: type: string description: Refers to the connector type for the staging table + path: + type: string + description: Refers to the path to the staging table + required: - connector - description: in the case of staging models, where an input source does not support direct write to the output and a staging table is required + description: | + in the case of staging models, where an input source does not support direct write to the output and a staging table is required + ```yaml + stage: + connector: s3 + path: s3://my_bucket/my_staging_table + ``` additionalProperties: true output: type: object From c55eff3c4dd9cf29cc64a1e2418bcd7a90e14fbf Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:58:35 -0600 Subject: [PATCH 25/32] god so many false positives from asking cursor to merge files --- docs/docs/hidden/yaml/canvas-dashboards.md | 74 ++-- docs/docs/hidden/yaml/explore-dashboards.md | 96 ++++-- docs/docs/hidden/yaml/metrics-views.md | 58 ++-- docs/docs/hidden/yaml/themes.md | 31 +- runtime/parser/schema/project.schema.yaml | 358 ++++++-------------- 5 files changed, 227 insertions(+), 390 deletions(-) diff --git a/docs/docs/hidden/yaml/canvas-dashboards.md b/docs/docs/hidden/yaml/canvas-dashboards.md index 2bb88bd4d0a..16d2ed54eaf 100644 --- a/docs/docs/hidden/yaml/canvas-dashboards.md +++ b/docs/docs/hidden/yaml/canvas-dashboards.md @@ -50,48 +50,6 @@ _[array of object]_ - Refers to all of the rows displayed on the Canvas - **`width`** - _[string, integer]_ - Width of the component (can be a number or string with unit) -### `components` - -_[array of object]_ - Array of components to display on the canvas - - - **`type`** - _[string]_ - Type of component (chart, table, text, etc.) - - - **`title`** - _[string]_ - Title for the component - - - **`data`** - _[oneOf]_ - Data source for the component - - - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - - - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. - - - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - - - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - - - **`args`** - _[object]_ - Arguments to pass to the custom API. - - - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - - - **option 1** - _[string]_ - A simple file path/glob pattern as a string. - - - **option 2** - _[object]_ - An object-based configuration for specifying a file path/glob pattern with advanced options. - - - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. - - - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - - - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. - -```yaml -resource_status: - where_error: true -``` - - - - **`layout`** - _[object]_ - Layout configuration for the component - - - **`style`** - _[object]_ - Styling configuration for the component - ### `max_width` _[integer]_ - Max width in pixels of the canvas @@ -114,7 +72,7 @@ _[boolean]_ - Defaults to true, when set to false it will hide the ability to se ### `time_ranges` -_[array of oneOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' +_[array of anyOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' ```yaml time_ranges: - PT15M // Simplified syntax to specify only the range @@ -128,9 +86,17 @@ _[array of oneOf]_ - Overrides the list of default time range selections availab ``` + - **option 1** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection + + - **option 2** - _[object]_ - Object containing time range and comparison configuration + - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ - - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + - **`comparison_offsets`** - _[array of anyOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + + - **option 1** - _[string]_ - Offset string only (range is inferred) + + - **option 2** - _[object]_ - Object containing offset and range configuration for time comparison - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) @@ -148,4 +114,22 @@ _[object]_ - Indicates if filters should be enabled for the canvas. _[object]_ - Defines security rules and access control policies for dashboards (without row filtering) - - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. \ No newline at end of file + - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + +## Common Properties + +### `name` + +_[string]_ - Name is usually inferred from the filename, but can be specified manually. + +### `refs` + +_[array of string]_ - List of resource references + +### `dev` + +_[object]_ - Overrides any properties in development environment. + +### `prod` + +_[object]_ - Overrides any properties in production environment. \ No newline at end of file diff --git a/docs/docs/hidden/yaml/explore-dashboards.md b/docs/docs/hidden/yaml/explore-dashboards.md index eaf82323f0d..2a9be0c5cd1 100644 --- a/docs/docs/hidden/yaml/explore-dashboards.md +++ b/docs/docs/hidden/yaml/explore-dashboards.md @@ -47,33 +47,33 @@ _[anyOf]_ - List of dimension names. Use '*' to select all dimensions (default) ``` - - **option 1** - _[string]_ - Simple field name as a string. + - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection - - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. + - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection - - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. + - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion - - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. + - **`regex`** - _[string]_ - Select dimensions using a regular expression - - **`name`** - _[string]_ - Name of the field to select. _(required)_ + - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. + - **`exclude`** - _[object]_ - Select all dimensions except those listed here ### `measures` _[anyOf]_ - List of measure names. Use '*' to select all measures (default) - - **option 1** - _[string]_ - Simple field name as a string. + - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection - - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. + - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection - - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. + - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion - - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. + - **`regex`** - _[string]_ - Select dimensions using a regular expression - - **`name`** - _[string]_ - Name of the field to select. _(required)_ + - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. + - **`exclude`** - _[object]_ - Select all dimensions except those listed here ### `theme` @@ -81,11 +81,19 @@ _[oneOf]_ - Name of the theme to use. Only one of theme and embedded_theme can b ### `time_range` -_[oneOf]_ - Default time range for the dashboard +_[anyOf]_ - Default time range for the dashboard + + - **option 1** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection + + - **option 2** - _[object]_ - Object containing time range and comparison configuration - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ - - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + - **`comparison_offsets`** - _[array of anyOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + + - **option 1** - _[string]_ - Offset string only (range is inferred) + + - **option 2** - _[object]_ - Object containing offset and range configuration for time comparison - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) @@ -93,7 +101,7 @@ _[oneOf]_ - Default time range for the dashboard ### `time_ranges` -_[array of oneOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' +_[array of anyOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' ```yaml time_ranges: - PT15M // Simplified syntax to specify only the range @@ -107,9 +115,17 @@ _[array of oneOf]_ - Overrides the list of default time range selections availab ``` + - **option 1** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection + + - **option 2** - _[object]_ - Object containing time range and comparison configuration + - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ - - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + - **`comparison_offsets`** - _[array of anyOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + + - **option 1** - _[string]_ - Offset string only (range is inferred) + + - **option 2** - _[object]_ - Object containing offset and range configuration for time comparison - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) @@ -146,31 +162,31 @@ _[object]_ - defines the defaults YAML struct - **`dimensions`** - _[anyOf]_ - Provides the default dimensions to load on viewing the dashboard - - **option 1** - _[string]_ - Simple field name as a string. + - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection - - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. + - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection - - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. + - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion - - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. + - **`regex`** - _[string]_ - Select dimensions using a regular expression - - **`name`** - _[string]_ - Name of the field to select. _(required)_ + - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. + - **`exclude`** - _[object]_ - Select all dimensions except those listed here - **`measures`** - _[anyOf]_ - Provides the default measures to load on viewing the dashboard - - **option 1** - _[string]_ - Simple field name as a string. + - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection - - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. + - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection - - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. + - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion - - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. + - **`regex`** - _[string]_ - Select dimensions using a regular expression - - **`name`** - _[string]_ - Name of the field to select. _(required)_ + - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. + - **`exclude`** - _[object]_ - Select all dimensions except those listed here - **`time_range`** - _[string]_ - Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) @@ -184,18 +200,26 @@ _[object]_ - Configuration options for embedded dashboard views - **`hide_pivot`** - _[boolean]_ - When true, hides the pivot table view in embedded mode -### `filters` +### `security` -_[array of object]_ - Default filters to apply to the dashboard +_[object]_ - Defines security rules and access control policies for dashboards (without row filtering) - - **`dimension`** - _[string]_ - Dimension to filter on + - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. - - **`value`** - _[no type]_ - Value to filter by +## Common Properties - - **`operator`** - _[string]_ - Filter operator (eq, ne, in, etc.) +### `name` -### `security` +_[string]_ - Name is usually inferred from the filename, but can be specified manually. -_[object]_ - Defines security rules and access control policies for dashboards (without row filtering) +### `refs` + +_[array of string]_ - List of resource references + +### `dev` + +_[object]_ - Overrides any properties in development environment. + +### `prod` - - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. \ No newline at end of file +_[object]_ - Overrides any properties in production environment. \ No newline at end of file diff --git a/docs/docs/hidden/yaml/metrics-views.md b/docs/docs/hidden/yaml/metrics-views.md index ae87481d88d..7547951a454 100644 --- a/docs/docs/hidden/yaml/metrics-views.md +++ b/docs/docs/hidden/yaml/metrics-views.md @@ -108,11 +108,47 @@ _[array of object]_ - Used to define the numeric aggregates of columns from your - **`order`** - _[string]_ - Specifies the fields to order the window by, determining the sequence of rows within each partition. + - **option 1** - _[string]_ - Simple field name as a string. + + - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. + + - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. + + - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. + + - **`name`** - _[string]_ - Name of the field to select. _(required)_ + + - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. + - **`frame`** - _[string]_ - Defines the window frame boundaries for calculations, specifying which rows are included in the window relative to the current row. - - **`per`** - _[string]_ - for per dimensions + - **`per`** - _[anyOf]_ - for per dimensions + + - **option 1** - _[string]_ - Simple field name as a string. + + - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. + + - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. + + - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. + + - **`name`** - _[string]_ - Name of the field to select. _(required)_ + + - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. - - **`requires`** - _[object]_ - using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures + - **`requires`** - _[anyOf]_ - using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures + + - **option 1** - _[string]_ - Simple field name as a string. + + - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. + + - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. + + - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. + + - **`name`** - _[string]_ - Name of the field to select. _(required)_ + + - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. - **`valid_percent_of_total`** - _[boolean]_ - a boolean indicating whether percent-of-total values should be rendered for this measure @@ -127,8 +163,6 @@ _[array of object]_ - Used to define the numeric aggregates of columns from your - `interval_ms`: Convert milliseconds into human-readable durations like hours (h), days (d), years (y), etc. (optional) - - **`format`** - _[string]_ - a custom format string for the measure - - **`format_d3`** - _[string]_ - Controls the formatting of this measure using a [d3-format](https://d3js.org/d3-format) string. If an invalid format string is supplied, the measure will fall back to `format_preset: humanize`. A measure cannot have both `format_preset` and `format_d3`. If neither is provided, the humanize preset is used by default. Example: `format_d3: ".2f"` formats using fixed-point notation with two decimal places. Example: `format_d3: ",.2r"` formats using grouped thousands with two significant digits. (optional) - **`format_d3_locale`** - _[object]_ - locale configuration passed through to D3, enabling changing the currency symbol among other things. For details, see the docs for D3's formatLocale. @@ -144,24 +178,8 @@ _[array of object]_ - Used to define the numeric aggregates of columns from your - **`currency`** - _[array]_ - the currency symbol - - **`decimals`** - _[integer]_ - number of decimal places to display - - - **`prefix`** - _[string]_ - text to display before the measure value - - - **`suffix`** - _[string]_ - text to display after the measure value - - - **`multiplier`** - _[number]_ - multiply the measure value by this number - - - **`hidden`** - _[boolean]_ - if true, the measure will not be displayed in the UI - - **`treat_nulls_as`** - _[string]_ - used to configure what value to fill in for missing time buckets. This also works generally as COALESCING over non empty time buckets. - - **`drill_through`** - _[object]_ - configuration for drill-through functionality - - - **`enabled`** - _[boolean]_ - whether drill-through is enabled for this measure - - - **`target`** - _[string]_ - the target dashboard or URL for drill-through - ### `annotations` _[array of object]_ - Used to define annotations that can be displayed on charts diff --git a/docs/docs/hidden/yaml/themes.md b/docs/docs/hidden/yaml/themes.md index 381748ee361..3186e7e929d 100644 --- a/docs/docs/hidden/yaml/themes.md +++ b/docs/docs/hidden/yaml/themes.md @@ -4,7 +4,10 @@ title: Theme YAML sidebar_position: 40 --- -Themes allow you to customize the appearance of your dashboards and UI components. +In your Rill project directory, create a `.yaml` file in any directory containing `type: theme`. Rill will automatically ingest the theme next time you run `rill start` or deploy to Rill Cloud. + +To apply that theme to a dashboard, add `default_theme: ` to the yaml file for that dashboard. Alternatively, you can add this to the end of the URL in your browser: `?theme=` + ## Properties @@ -12,14 +15,6 @@ Themes allow you to customize the appearance of your dashboards and UI component _[string]_ - Refers to the resource type and must be `theme` _(required)_ -### `display_name` - -_[string]_ - Display name for the theme _(required)_ - -### `description` - -_[string]_ - Description for the theme - ### `colors` _[object]_ - Color palette for the theme @@ -28,24 +23,6 @@ _[object]_ - Color palette for the theme - **`secondary`** - _[string]_ - Secondary color - - **`accent`** - _[string]_ - Accent color - - - **`background`** - _[string]_ - Background color - - - **`text`** - _[string]_ - Text color - -### `fonts` - -_[object]_ - Font configuration for the theme - - - **`family`** - _[string]_ - Font family - - - **`size`** - _[string]_ - Base font size - -### `spacing` - -_[object]_ - Spacing configuration for the theme - ## Common Properties ### `name` diff --git a/runtime/parser/schema/project.schema.yaml b/runtime/parser/schema/project.schema.yaml index b75426f188a..e86c72585b7 100644 --- a/runtime/parser/schema/project.schema.yaml +++ b/runtime/parser/schema/project.schema.yaml @@ -532,18 +532,18 @@ oneOf: description: 'Controls whether the window is partitioned. When true, calculations are performed within each partition separately.' order: type: string + $ref: '#/definitions/field_selectors_properties' description: 'Specifies the fields to order the window by, determining the sequence of rows within each partition.' frame: type: string description: 'Defines the window frame boundaries for calculations, specifying which rows are included in the window relative to the current row.' additionalProperties: false per: - type: string + $ref: '#/definitions/field_selectors_properties' description: for per dimensions requires: - type: object + $ref: '#/definitions/field_selectors_properties' description: using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures - additionalProperties: true valid_percent_of_total: type: boolean description: a boolean indicating whether percent-of-total values should be rendered for this measure @@ -559,9 +559,6 @@ oneOf: - `currency_eur`: Round to 2 decimal points with a euro sign (€). - `percentage`: Convert a rate into a percentage with a % sign. - `interval_ms`: Convert milliseconds into human-readable durations like hours (h), days (d), years (y), etc. (optional) - format: - type: string - description: a custom format string for the measure format_d3: type: string description: 'Controls the formatting of this measure using a [d3-format](https://d3js.org/d3-format) string. If an invalid format string is supplied, the measure will fall back to `format_preset: humanize`. A measure cannot have both `format_preset` and `format_d3`. If neither is provided, the humanize preset is used by default. Example: `format_d3: ".2f"` formats using fixed-point notation with two decimal places. Example: `format_d3: ",.2r"` formats using grouped thousands with two significant digits. (optional)' @@ -582,34 +579,11 @@ oneOf: currency: type: array description: the currency symbol - decimals: - type: integer - description: number of decimal places to display - prefix: - type: string - description: text to display before the measure value - suffix: - type: string - description: text to display after the measure value - multiplier: - type: number - description: multiply the measure value by this number - hidden: - type: boolean - description: if true, the measure will not be displayed in the UI + treat_nulls_as: type: string description: used to configure what value to fill in for missing time buckets. This also works generally as COALESCING over non empty time buckets. - drill_through: - type: object - description: configuration for drill-through functionality - properties: - enabled: - type: boolean - description: whether drill-through is enabled for this measure - target: - type: string - description: the target dashboard or URL for drill-through + required: - name - display_name @@ -751,29 +725,7 @@ oneOf: description: Width of the component (can be a number or string with unit) additionalProperties: true additionalProperties: false - components: - type: array - description: Array of components to display on the canvas - items: - type: object - properties: - type: - type: string - description: Type of component (chart, table, text, etc.) - title: - type: string - description: Title for the component - data: - $ref: '#/definitions/data_properties' - description: Data source for the component - layout: - type: object - description: Layout configuration for the component - additionalProperties: true - style: - type: object - description: Styling configuration for the component - additionalProperties: true + max_width: type: integer description: Max width in pixels of the canvas @@ -830,6 +782,7 @@ oneOf: required: - type - display_name + - $ref: '#/definitions/common_properties' # Explore Dashboards - title: Explore Dashboard YAML @@ -873,88 +826,10 @@ oneOf: dimensions: regex: "^public_.*$" ``` - anyOf: - - type: string - description: Simple field name as a string. - - type: array - description: List of field selectors, each can be a string or an object with detailed configuration. - items: - anyOf: - - type: string - description: Shorthand field selector, interpreted as the name. - - type: object - description: Detailed field selector configuration with name and optional time grain. - properties: - name: - type: string - description: Name of the field to select. - time_grain: - type: string - description: Time grain for time-based dimensions. - enum: - - '' - - ms - - millisecond - - s - - second - - min - - minute - - h - - hour - - d - - day - - w - - week - - month - - q - - quarter - - 'y' - - year - required: - - name - additionalProperties: false + $ref: '#/definitions/field_selector_properties' measures: description: List of measure names. Use '*' to select all measures (default) - anyOf: - - type: string - description: Simple field name as a string. - - type: array - description: List of field selectors, each can be a string or an object with detailed configuration. - items: - anyOf: - - type: string - description: Shorthand field selector, interpreted as the name. - - type: object - description: Detailed field selector configuration with name and optional time grain. - properties: - name: - type: string - description: Name of the field to select. - time_grain: - type: string - description: Time grain for time-based dimensions. - enum: - - '' - - ms - - millisecond - - s - - second - - min - - minute - - h - - hour - - d - - day - - w - - week - - month - - q - - quarter - - 'y' - - year - required: - - name - additionalProperties: false + $ref: '#/definitions/field_selector_properties' theme: oneOf: - title: Existing theme @@ -1014,88 +889,10 @@ oneOf: properties: dimensions: description: Provides the default dimensions to load on viewing the dashboard - anyOf: - - type: string - description: Simple field name as a string. - - type: array - description: List of field selectors, each can be a string or an object with detailed configuration. - items: - anyOf: - - type: string - description: Shorthand field selector, interpreted as the name. - - type: object - description: Detailed field selector configuration with name and optional time grain. - properties: - name: - type: string - description: Name of the field to select. - time_grain: - type: string - description: Time grain for time-based dimensions. - enum: - - '' - - ms - - millisecond - - s - - second - - min - - minute - - h - - hour - - d - - day - - w - - week - - month - - q - - quarter - - 'y' - - year - required: - - name - additionalProperties: false + $ref: '#/definitions/field_selector_properties' measures: description: Provides the default measures to load on viewing the dashboard - anyOf: - - type: string - description: Simple field name as a string. - - type: array - description: List of field selectors, each can be a string or an object with detailed configuration. - items: - anyOf: - - type: string - description: Shorthand field selector, interpreted as the name. - - type: object - description: Detailed field selector configuration with name and optional time grain. - properties: - name: - type: string - description: Name of the field to select. - time_grain: - type: string - description: Time grain for time-based dimensions. - enum: - - '' - - ms - - millisecond - - s - - second - - min - - minute - - h - - hour - - d - - day - - w - - week - - month - - q - - quarter - - 'y' - - year - required: - - name - additionalProperties: false + $ref: '#/definitions/field_selector_properties' time_range: description: Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) type: string @@ -1114,20 +911,7 @@ oneOf: type: boolean description: When true, hides the pivot table view in embedded mode additionalProperties: false - filters: - type: array - description: Default filters to apply to the dashboard - items: - type: object - properties: - dimension: - type: string - description: Dimension to filter on - value: - description: Value to filter by - operator: - type: string - description: Filter operator (eq, ne, in, etc.) + security: description: Security rules to apply for access to the explore dashboard $ref: '#/definitions/dashboard_security_policy_properties' @@ -1135,6 +919,7 @@ oneOf: - type - display_name - metrics_view + - $ref: '#/definitions/common_properties' # Alerts - title: Alert YAML @@ -1338,7 +1123,11 @@ oneOf: - title: Theme YAML id: themes type: object - description: Themes allow you to customize the appearance of your dashboards and UI components. + description: | + In your Rill project directory, create a `.yaml` file in any directory containing `type: theme`. Rill will automatically ingest the theme next time you run `rill start` or deploy to Rill Cloud. + + To apply that theme to a dashboard, add `default_theme: ` to the yaml file for that dashboard. Alternatively, you can add this to the end of the URL in your browser: `?theme=` + allOf: - title: Properties type: object @@ -1347,12 +1136,6 @@ oneOf: type: string const: theme description: Refers to the resource type and must be `theme` - display_name: - type: string - description: Display name for the theme - description: - type: string - description: Description for the theme colors: type: object description: Color palette for the theme @@ -1363,31 +1146,8 @@ oneOf: secondary: type: string description: Secondary color - accent: - type: string - description: Accent color - background: - type: string - description: Background color - text: - type: string - description: Text color - additionalProperties: true - fonts: - type: object - description: Font configuration for the theme - properties: - family: - type: string - description: Font family - size: - type: string - description: Base font size - additionalProperties: true - spacing: - type: object - description: Spacing configuration for the theme additionalProperties: true + required: - type - display_name @@ -1734,7 +1494,7 @@ definitions: where_error: true ``` explore_time_range_properties: - oneOf: + anyOf: - type: string description: a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection - type: object @@ -1747,7 +1507,7 @@ definitions: type: array description: list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) items: - oneOf: + anyOf: - type: string description: Offset string only (range is inferred) - type: object @@ -2598,4 +2358,78 @@ definitions: - driver - dsn - driver - - dsn \ No newline at end of file + - dsn + field_selectors_properties: + anyOf: + - type: string + description: 'Simple field name as a string.' + - type: array + description: 'List of field selectors, each can be a string or an object with detailed configuration.' + items: + anyOf: + - type: string + description: 'Shorthand field selector, interpreted as the name.' + - type: object + description: 'Detailed field selector configuration with name and optional time grain.' + properties: + name: + type: string + description: 'Name of the field to select.' + time_grain: + type: string + description: 'Time grain for time-based dimensions.' + enum: + - '' + - ms + - millisecond + - s + - second + - min + - minute + - h + - hour + - d + - day + - w + - week + - month + - q + - quarter + - 'y' + - year + required: + - name + additionalProperties: false + minItems: 1 + field_selector_properties: + anyOf: + - title: Wildcard(*) selector + type: string + const: '*' + description: Wildcard(*) selector that includes all available fields in the selection + - title: Explicit list of fields + type: array + items: + type: string + description: Explicit list of fields to include in the selection + - title: Regex matching + type: object + description: 'Advanced matching using regex, DuckDB expression, or exclusion' + properties: + regex: + type: string + description: Select dimensions using a regular expression + expr: + type: string + description: DuckDB SQL expression to select fields based on custom logic + exclude: + type: object + description: Select all dimensions except those listed here + additionalProperties: false + oneOf: + - required: + - regex + - required: + - expr + - required: + - exclude \ No newline at end of file From 6ffe7f88f17df796e18be7fcac2f0630e73b6d6b Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Wed, 6 Aug 2025 17:12:10 -0600 Subject: [PATCH 26/32] adding links --- docs/docs/hidden/yaml/apis.md | 21 +-------------------- docs/docs/hidden/yaml/canvas-dashboards.md | 2 +- docs/docs/hidden/yaml/explore-dashboards.md | 2 +- docs/docs/hidden/yaml/metrics-views.md | 10 +++++++--- docs/docs/hidden/yaml/rillyaml.md | 4 ++-- runtime/parser/schema/project.schema.yaml | 17 ++++++++++------- 6 files changed, 22 insertions(+), 34 deletions(-) diff --git a/docs/docs/hidden/yaml/apis.md b/docs/docs/hidden/yaml/apis.md index 798f43450ea..cd5ace60d67 100644 --- a/docs/docs/hidden/yaml/apis.md +++ b/docs/docs/hidden/yaml/apis.md @@ -26,7 +26,7 @@ _[object]_ - OpenAPI specification for the API endpoint ### `security` -_[object]_ - Defines security rules and access control policies for resources +_[object]_ - Defines [security rules and access control policies](/manage/security) for resources - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. @@ -162,22 +162,3 @@ _[object]_ - Based on resource status _(required)_ resource_status: where_error: true ``` - - -## Common Properties - -### `name` - -_[string]_ - Name is usually inferred from the filename, but can be specified manually. - -### `refs` - -_[array of string]_ - List of resource references - -### `dev` - -_[object]_ - Overrides any properties in development environment. - -### `prod` - -_[object]_ - Overrides any properties in production environment. \ No newline at end of file diff --git a/docs/docs/hidden/yaml/canvas-dashboards.md b/docs/docs/hidden/yaml/canvas-dashboards.md index 16d2ed54eaf..4d37d7a54e1 100644 --- a/docs/docs/hidden/yaml/canvas-dashboards.md +++ b/docs/docs/hidden/yaml/canvas-dashboards.md @@ -112,7 +112,7 @@ _[object]_ - Indicates if filters should be enabled for the canvas. ### `security` -_[object]_ - Defines security rules and access control policies for dashboards (without row filtering) +_[object]_ - Defines [security rules and access control policies](/manage/security) for dashboards (without row filtering) - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. diff --git a/docs/docs/hidden/yaml/explore-dashboards.md b/docs/docs/hidden/yaml/explore-dashboards.md index 2a9be0c5cd1..40a2a892b87 100644 --- a/docs/docs/hidden/yaml/explore-dashboards.md +++ b/docs/docs/hidden/yaml/explore-dashboards.md @@ -202,7 +202,7 @@ _[object]_ - Configuration options for embedded dashboard views ### `security` -_[object]_ - Defines security rules and access control policies for dashboards (without row filtering) +_[object]_ - Defines [security rules and access control policies](/manage/security) for dashboards (without row filtering) - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. diff --git a/docs/docs/hidden/yaml/metrics-views.md b/docs/docs/hidden/yaml/metrics-views.md index 7547951a454..b0641622dc6 100644 --- a/docs/docs/hidden/yaml/metrics-views.md +++ b/docs/docs/hidden/yaml/metrics-views.md @@ -16,6 +16,10 @@ _[string]_ - The version of the metrics view schema _[string]_ - Refers to the resource type and must be `metrics_view` _(required)_ +### `connector` + +_[string]_ - Refers to the connector type for the metrics view, see [OLAP engines](/build/olap) for more information + ### `display_name` _[string]_ - Refers to the display name for the metrics view @@ -26,7 +30,7 @@ _[string]_ - Refers to the description for the metrics view ### `ai_instructions` -_[string]_ - Extra instructions for AI agents. Used to guide natural language question answering and routing. +_[string]_ - Extra instructions for [AI agents](/explore/mcp). Used to guide natural language question answering and routing. ### `model` @@ -136,7 +140,7 @@ _[array of object]_ - Used to define the numeric aggregates of columns from your - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. - - **`requires`** - _[anyOf]_ - using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures + - **`requires`** - _[anyOf]_ - using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures. See [referencing measures](/build/metrics-view/advanced-expressions/referencing) for more information. - **option 1** - _[string]_ - Simple field name as a string. @@ -212,7 +216,7 @@ _[array of object]_ - Used to define annotations that can be displayed on charts ### `security` -_[object]_ - Defines security rules and access control policies for resources +_[object]_ - Defines [security rules and access control policies](/manage/security) for resources - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. diff --git a/docs/docs/hidden/yaml/rillyaml.md b/docs/docs/hidden/yaml/rillyaml.md index 11908c021fe..ecf2bc59917 100644 --- a/docs/docs/hidden/yaml/rillyaml.md +++ b/docs/docs/hidden/yaml/rillyaml.md @@ -26,7 +26,7 @@ _[object]_ - Optional feature flags. Can be specified as a map of feature names ### `ai_instructions` -_[string]_ - Extra instructions for LLM/AI features. Used to guide natural language question answering and routing. +_[string]_ - Extra instructions for [AI agents](/explore/mcp). Used to guide natural language question answering and routing. ## Configuring the default OLAP Engine @@ -38,7 +38,7 @@ Please see our reference documentation on [OLAP Engines](/docs/reference/olap-en ### `olap_connector` -_[string]_ - Specifies the default OLAP engine for the project. Defaults to duckdb if not set. +_[string]_ - Specifies the [default OLAP engine](/build/olap) for the project. Defaults to duckdb if not set. ```yaml olap_connector: clickhouse diff --git a/runtime/parser/schema/project.schema.yaml b/runtime/parser/schema/project.schema.yaml index e86c72585b7..a25987efb23 100644 --- a/runtime/parser/schema/project.schema.yaml +++ b/runtime/parser/schema/project.schema.yaml @@ -421,6 +421,9 @@ oneOf: type: string const: metrics_view description: Refers to the resource type and must be `metrics_view` + connector: + type: string + description: Refers to the connector type for the metrics view, see [OLAP engines](/build/olap) for more information display_name: type: string description: Refers to the display name for the metrics view @@ -429,7 +432,7 @@ oneOf: description: Refers to the description for the metrics view ai_instructions: type: string - description: Extra instructions for AI agents. Used to guide natural language question answering and routing. + description: Extra instructions for [AI agents](/explore/mcp). Used to guide natural language question answering and routing. model: type: string description: Refers to the model powering the dashboard (either model or table is required) @@ -543,7 +546,7 @@ oneOf: description: for per dimensions requires: $ref: '#/definitions/field_selectors_properties' - description: using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures + description: using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures. See [referencing measures](/build/metrics-view/advanced-expressions/referencing) for more information. valid_percent_of_total: type: boolean description: a boolean indicating whether percent-of-total values should be rendered for this measure @@ -1117,7 +1120,7 @@ oneOf: required: - type - $ref: '#/definitions/data_properties' - - $ref: '#/definitions/common_properties' + # - $ref: '#/definitions/common_properties' # Themes - title: Theme YAML @@ -1176,7 +1179,7 @@ oneOf: description: Optional feature flags. Can be specified as a map of feature names to booleans. ai_instructions: type: string - description: Extra instructions for LLM/AI features. Used to guide natural language question answering and routing. + description: Extra instructions for [AI agents](/explore/mcp). Used to guide natural language question answering and routing. - title: Configuring the default OLAP Engine description: | Rill allows you to specify the default OLAP engine to use in your project via `rill.yaml`. @@ -1187,7 +1190,7 @@ oneOf: properties: olap_connector: type: string - description: Specifies the default OLAP engine for the project. Defaults to duckdb if not set. + description: Specifies the [default OLAP engine](/build/olap) for the project. Defaults to duckdb if not set. examples: - olap_connector: clickhouse - title: Project-wide defaults @@ -1592,7 +1595,7 @@ definitions: description: 'If true, allows the schedule to run in development mode.' security_policy_properties: type: object - description: Defines security rules and access control policies for resources + description: Defines [security rules and access control policies](/manage/security) for resources properties: access: oneOf: @@ -1689,7 +1692,7 @@ definitions: dashboard_security_policy_properties: type: object - description: Defines security rules and access control policies for dashboards (without row filtering) + description: Defines [security rules and access control policies](/manage/security) for dashboards (without row filtering) properties: access: oneOf: From f69c89ae1d7f1118dd2ab7c0d2e7e6f5d64c45b4 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Wed, 6 Aug 2025 17:23:00 -0600 Subject: [PATCH 27/32] fixed oneOf --- cli/cmd/docs/generate_project.go | 3 +- docs/docs/hidden/yaml/advanced-models.md | 76 +++++++++++++++++++++ docs/docs/hidden/yaml/alerts.md | 44 ++++++++++++ docs/docs/hidden/yaml/apis.md | 42 ++++++++++++ docs/docs/hidden/yaml/canvas-dashboards.md | 12 +++- docs/docs/hidden/yaml/explore-dashboards.md | 28 +++++--- docs/docs/hidden/yaml/metrics-views.md | 14 ++-- runtime/parser/schema/project.schema.yaml | 14 ++-- 8 files changed, 208 insertions(+), 25 deletions(-) diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index 79a05f5c41e..faf34438c76 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -491,8 +491,9 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req doc.WriteString("\n\n## One of Properties Options") } - for _, item := range oneOf.Content { + for i, item := range oneOf.Content { if hasType(item) || hasProperties(item) || hasCombinators(item) { + doc.WriteString(fmt.Sprintf("\n\n%s- **option %d** - %s - %s", indent, i+1, getPrintableType(item), getPrintableDescription(item, indent, "(no description)"))) doc.WriteString(generateDoc(sidebarPosition, level, item, indent+" ", getRequiredMapFromNode(item), rootSchema, id)) } diff --git a/docs/docs/hidden/yaml/advanced-models.md b/docs/docs/hidden/yaml/advanced-models.md index 3d717bad67f..4d1663b93cc 100644 --- a/docs/docs/hidden/yaml/advanced-models.md +++ b/docs/docs/hidden/yaml/advanced-models.md @@ -93,16 +93,52 @@ state: ``` + - **option 1** - _[object]_ - Executes a raw SQL query against the project's data models. + ```yaml + type: api + + connector: "duckdb" + sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" + ``` + + - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. + - **option 2** - _[object]_ - Executes a SQL query that targets a defined metrics view. + ```yaml + type: api + + metrics_sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" + ``` + + - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ + - **option 3** - _[object]_ - Calls a custom API defined in the project to compute data. + ```yaml + type: api + + api: "user_analytics_api" + args: + start_date: "2024-01-01" + limit: 10 + ``` + + - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - **`args`** - _[object]_ - Arguments to pass to the custom API. + - **option 4** - _[object]_ - Uses a file-matching pattern (glob) to query data from a connector. + ```yaml + type: api + + glob: "data/*.csv" + ``` + + - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - **option 1** - _[string]_ - A simple file path/glob pattern as a string. @@ -111,6 +147,8 @@ state: - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. + - **option 5** - _[object]_ - Uses the status of a resource as data. + - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. @@ -135,16 +173,52 @@ partitions: ``` + - **option 1** - _[object]_ - Executes a raw SQL query against the project's data models. + ```yaml + type: api + + connector: "duckdb" + sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" + ``` + + - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. + - **option 2** - _[object]_ - Executes a SQL query that targets a defined metrics view. + ```yaml + type: api + + metrics_sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" + ``` + + - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ + - **option 3** - _[object]_ - Calls a custom API defined in the project to compute data. + ```yaml + type: api + + api: "user_analytics_api" + args: + start_date: "2024-01-01" + limit: 10 + ``` + + - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - **`args`** - _[object]_ - Arguments to pass to the custom API. + - **option 4** - _[object]_ - Uses a file-matching pattern (glob) to query data from a connector. + ```yaml + type: api + + glob: "data/*.csv" + ``` + + - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - **option 1** - _[string]_ - A simple file path/glob pattern as a string. @@ -153,6 +227,8 @@ partitions: - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. + - **option 5** - _[object]_ - Uses the status of a resource as data. + - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. diff --git a/docs/docs/hidden/yaml/alerts.md b/docs/docs/hidden/yaml/alerts.md index 2643a3503e2..c3152e6fcaf 100644 --- a/docs/docs/hidden/yaml/alerts.md +++ b/docs/docs/hidden/yaml/alerts.md @@ -44,16 +44,52 @@ _[string]_ - Description for the alert _[oneOf]_ - Data source for the alert _(required)_ + - **option 1** - _[object]_ - Executes a raw SQL query against the project's data models. + ```yaml + type: api + + connector: "duckdb" + sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" + ``` + + - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. + - **option 2** - _[object]_ - Executes a SQL query that targets a defined metrics view. + ```yaml + type: api + + metrics_sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" + ``` + + - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ + - **option 3** - _[object]_ - Calls a custom API defined in the project to compute data. + ```yaml + type: api + + api: "user_analytics_api" + args: + start_date: "2024-01-01" + limit: 10 + ``` + + - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - **`args`** - _[object]_ - Arguments to pass to the custom API. + - **option 4** - _[object]_ - Uses a file-matching pattern (glob) to query data from a connector. + ```yaml + type: api + + glob: "data/*.csv" + ``` + + - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - **option 1** - _[string]_ - A simple file path/glob pattern as a string. @@ -62,6 +98,8 @@ _[oneOf]_ - Data source for the alert _(required)_ - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. + - **option 5** - _[object]_ - Uses the status of a resource as data. + - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. @@ -94,10 +132,16 @@ _[string]_ - define the timeout of the alert in seconds (optional). _[oneOf]_ - Specifies how user identity or attributes should be evaluated for security policy enforcement. + - **option 1** - _[object]_ - Specifies a unique user identifier for applying security policies. + - **`user_id`** - _[string]_ - The unique user ID used to evaluate security policies. _(required)_ + - **option 2** - _[object]_ - Specifies a user's email address for applying security policies. + - **`user_email`** - _[string]_ - The user's email address used to evaluate security policies. _(required)_ + - **option 3** - _[object]_ - Specifies a set of arbitrary user attributes for applying security policies. + - **`attributes`** - _[object]_ - A dictionary of user attributes used to evaluate security policies. _(required)_ ### `on_recover` diff --git a/docs/docs/hidden/yaml/apis.md b/docs/docs/hidden/yaml/apis.md index cd5ace60d67..eb6e6c41db3 100644 --- a/docs/docs/hidden/yaml/apis.md +++ b/docs/docs/hidden/yaml/apis.md @@ -30,6 +30,10 @@ _[object]_ - Defines [security rules and access control policies](/manage/securi - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + - **option 1** - _[string]_ - SQL expression that evaluates to a boolean to determine access + + - **option 2** - _[boolean]_ - Direct boolean value to allow or deny access + - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded @@ -72,6 +76,15 @@ _[boolean]_ - Flag to control security inheritance ## One of Properties Options +- **option 1** - _[object]_ - Executes a raw SQL query against the project's data models. +```yaml +type: api + +connector: "duckdb" +sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" +``` + + ## SQL Query Executes a raw SQL query against the project's data models. @@ -91,6 +104,14 @@ _[string]_ - Raw SQL query to run against existing models in the project. _(requ _[string]_ - specifies the connector to use when running SQL or glob queries. +- **option 2** - _[object]_ - Executes a SQL query that targets a defined metrics view. + ```yaml + type: api + + metrics_sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" + ``` + + ## Metrics View Query Executes a SQL query that targets a defined metrics view. @@ -105,6 +126,17 @@ Executes a SQL query that targets a defined metrics view. _[string]_ - SQL query that targets a metrics view in the project _(required)_ +- **option 3** - _[object]_ - Calls a custom API defined in the project to compute data. + ```yaml + type: api + + api: "user_analytics_api" + args: + start_date: "2024-01-01" + limit: 10 + ``` + + ## Custom API Call Calls a custom API defined in the project to compute data. @@ -126,6 +158,14 @@ _[string]_ - Name of a custom API defined in the project. _(required)_ _[object]_ - Arguments to pass to the custom API. +- **option 4** - _[object]_ - Uses a file-matching pattern (glob) to query data from a connector. + ```yaml + type: api + + glob: "data/*.csv" + ``` + + ## File Glob Query Uses a file-matching pattern (glob) to query data from a connector. @@ -148,6 +188,8 @@ _[anyOf]_ - Defines the file path or pattern to query from the specified connect _[string]_ - Specifies the connector to use with the glob input. +- **option 5** - _[object]_ - Uses the status of a resource as data. + ## Resource Status Check Uses the status of a resource as data. diff --git a/docs/docs/hidden/yaml/canvas-dashboards.md b/docs/docs/hidden/yaml/canvas-dashboards.md index 4d37d7a54e1..f1eb35de023 100644 --- a/docs/docs/hidden/yaml/canvas-dashboards.md +++ b/docs/docs/hidden/yaml/canvas-dashboards.md @@ -66,13 +66,17 @@ _[integer]_ - Vertical gap in pixels of the canvas _[oneOf]_ - Theme configuration. Can be either a string reference to an existing theme or an inline theme configuration object. + - **option 1** - _[string]_ - Name of an existing theme to apply to the dashboard + + - **option 2** - _[object]_ - Inline theme configuration. + ### `allow_custom_time_range` _[boolean]_ - Defaults to true, when set to false it will hide the ability to set a custom time range for the user. ### `time_ranges` -_[array of anyOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' +_[array of oneOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' ```yaml time_ranges: - PT15M // Simplified syntax to specify only the range @@ -92,7 +96,7 @@ _[array of anyOf]_ - Overrides the list of default time range selections availab - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ - - **`comparison_offsets`** - _[array of anyOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) - **option 1** - _[string]_ - Offset string only (range is inferred) @@ -116,6 +120,10 @@ _[object]_ - Defines [security rules and access control policies](/manage/securi - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + - **option 1** - _[string]_ - SQL expression that evaluates to a boolean to determine access + + - **option 2** - _[boolean]_ - Direct boolean value to allow or deny access + ## Common Properties ### `name` diff --git a/docs/docs/hidden/yaml/explore-dashboards.md b/docs/docs/hidden/yaml/explore-dashboards.md index 40a2a892b87..352f64671ed 100644 --- a/docs/docs/hidden/yaml/explore-dashboards.md +++ b/docs/docs/hidden/yaml/explore-dashboards.md @@ -30,7 +30,7 @@ _[string]_ - Refers to the custom banner displayed at the header of an explore d ### `dimensions` -_[anyOf]_ - List of dimension names. Use '*' to select all dimensions (default) +_[oneOf]_ - List of dimension names. Use '*' to select all dimensions (default) ```yaml # Example: Select a dimension dimensions: @@ -61,7 +61,7 @@ _[anyOf]_ - List of dimension names. Use '*' to select all dimensions (default) ### `measures` -_[anyOf]_ - List of measure names. Use '*' to select all measures (default) +_[oneOf]_ - List of measure names. Use '*' to select all measures (default) - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection @@ -79,9 +79,13 @@ _[anyOf]_ - List of measure names. Use '*' to select all measures (default) _[oneOf]_ - Name of the theme to use. Only one of theme and embedded_theme can be set. -### `time_range` + - **option 1** - _[string]_ - Name of an existing theme to apply to the dashboard -_[anyOf]_ - Default time range for the dashboard + - **option 2** - _[object]_ - Inline theme configuration. + +### `time_ranges` + +_[oneOf]_ - Default time range for the dashboard - **option 1** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection @@ -89,7 +93,7 @@ _[anyOf]_ - Default time range for the dashboard - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ - - **`comparison_offsets`** - _[array of anyOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) - **option 1** - _[string]_ - Offset string only (range is inferred) @@ -101,7 +105,7 @@ _[anyOf]_ - Default time range for the dashboard ### `time_ranges` -_[array of anyOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' +_[array of oneOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' ```yaml time_ranges: - PT15M // Simplified syntax to specify only the range @@ -121,7 +125,7 @@ _[array of anyOf]_ - Overrides the list of default time range selections availab - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ - - **`comparison_offsets`** - _[array of anyOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) - **option 1** - _[string]_ - Offset string only (range is inferred) @@ -160,7 +164,7 @@ _[object]_ - defines the defaults YAML struct ``` - - **`dimensions`** - _[anyOf]_ - Provides the default dimensions to load on viewing the dashboard + - **`dimensions`** - _[oneOf]_ - Provides the default dimensions to load on viewing the dashboard - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection @@ -174,7 +178,7 @@ _[object]_ - defines the defaults YAML struct - **`exclude`** - _[object]_ - Select all dimensions except those listed here - - **`measures`** - _[anyOf]_ - Provides the default measures to load on viewing the dashboard + - **`measures`** - _[oneOf]_ - Provides the default measures to load on viewing the dashboard - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection @@ -188,7 +192,7 @@ _[object]_ - defines the defaults YAML struct - **`exclude`** - _[object]_ - Select all dimensions except those listed here - - **`time_range`** - _[string]_ - Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + - **`time_ranges`** - _[string]_ - Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) - **`comparison_mode`** - _[string]_ - Controls how to compare current data with historical or categorical baselines. Options: `none` (no comparison), `time` (compares with past based on default_time_range), `dimension` (compares based on comparison_dimension values) @@ -206,6 +210,10 @@ _[object]_ - Defines [security rules and access control policies](/manage/securi - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + - **option 1** - _[string]_ - SQL expression that evaluates to a boolean to determine access + + - **option 2** - _[boolean]_ - Direct boolean value to allow or deny access + ## Common Properties ### `name` diff --git a/docs/docs/hidden/yaml/metrics-views.md b/docs/docs/hidden/yaml/metrics-views.md index b0641622dc6..c9a7807a259 100644 --- a/docs/docs/hidden/yaml/metrics-views.md +++ b/docs/docs/hidden/yaml/metrics-views.md @@ -114,7 +114,7 @@ _[array of object]_ - Used to define the numeric aggregates of columns from your - **option 1** - _[string]_ - Simple field name as a string. - - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. + - **option 2** - _[array of oneOf]_ - List of field selectors, each can be a string or an object with detailed configuration. - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. @@ -126,11 +126,11 @@ _[array of object]_ - Used to define the numeric aggregates of columns from your - **`frame`** - _[string]_ - Defines the window frame boundaries for calculations, specifying which rows are included in the window relative to the current row. - - **`per`** - _[anyOf]_ - for per dimensions + - **`per`** - _[oneOf]_ - for per dimensions - **option 1** - _[string]_ - Simple field name as a string. - - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. + - **option 2** - _[array of oneOf]_ - List of field selectors, each can be a string or an object with detailed configuration. - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. @@ -140,11 +140,11 @@ _[array of object]_ - Used to define the numeric aggregates of columns from your - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. - - **`requires`** - _[anyOf]_ - using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures. See [referencing measures](/build/metrics-view/advanced-expressions/referencing) for more information. + - **`requires`** - _[oneOf]_ - using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures. See [referencing measures](/build/metrics-view/advanced-expressions/referencing) for more information. - **option 1** - _[string]_ - Simple field name as a string. - - **option 2** - _[array of anyOf]_ - List of field selectors, each can be a string or an object with detailed configuration. + - **option 2** - _[array of oneOf]_ - List of field selectors, each can be a string or an object with detailed configuration. - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. @@ -220,6 +220,10 @@ _[object]_ - Defines [security rules and access control policies](/manage/securi - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + - **option 1** - _[string]_ - SQL expression that evaluates to a boolean to determine access + + - **option 2** - _[boolean]_ - Direct boolean value to allow or deny access + - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded diff --git a/runtime/parser/schema/project.schema.yaml b/runtime/parser/schema/project.schema.yaml index a25987efb23..33183edf919 100644 --- a/runtime/parser/schema/project.schema.yaml +++ b/runtime/parser/schema/project.schema.yaml @@ -842,7 +842,7 @@ oneOf: type: object description: Inline theme configuration. description: Name of the theme to use. Only one of theme and embedded_theme can be set. - time_range: + time_ranges: $ref: '#/definitions/explore_time_range_properties' description: Default time range for the dashboard time_ranges: @@ -896,7 +896,7 @@ oneOf: measures: description: Provides the default measures to load on viewing the dashboard $ref: '#/definitions/field_selector_properties' - time_range: + time_ranges: description: Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) type: string comparison_mode: @@ -1497,7 +1497,7 @@ definitions: where_error: true ``` explore_time_range_properties: - anyOf: + oneOf: - type: string description: a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection - type: object @@ -1510,7 +1510,7 @@ definitions: type: array description: list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) items: - anyOf: + oneOf: - type: string description: Offset string only (range is inferred) - type: object @@ -2363,13 +2363,13 @@ definitions: - driver - dsn field_selectors_properties: - anyOf: + oneOf: - type: string description: 'Simple field name as a string.' - type: array description: 'List of field selectors, each can be a string or an object with detailed configuration.' items: - anyOf: + oneOf: - type: string description: 'Shorthand field selector, interpreted as the name.' - type: object @@ -2405,7 +2405,7 @@ definitions: additionalProperties: false minItems: 1 field_selector_properties: - anyOf: + oneOf: - title: Wildcard(*) selector type: string const: '*' From 3b6517e6e5b05de5ad3f0dfb5e568ac7e5905166 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Wed, 6 Aug 2025 17:25:45 -0600 Subject: [PATCH 28/32] split data properties to have api excamples --- docs/docs/hidden/yaml/advanced-models.md | 56 --------------- docs/docs/hidden/yaml/alerts.md | 28 -------- runtime/parser/schema/project.schema.yaml | 83 ++++++++++++++++++++++- 3 files changed, 81 insertions(+), 86 deletions(-) diff --git a/docs/docs/hidden/yaml/advanced-models.md b/docs/docs/hidden/yaml/advanced-models.md index 4d1663b93cc..42b9177c3d6 100644 --- a/docs/docs/hidden/yaml/advanced-models.md +++ b/docs/docs/hidden/yaml/advanced-models.md @@ -94,50 +94,22 @@ state: - **option 1** - _[object]_ - Executes a raw SQL query against the project's data models. - ```yaml - type: api - - connector: "duckdb" - sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" - ``` - - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. - **option 2** - _[object]_ - Executes a SQL query that targets a defined metrics view. - ```yaml - type: api - - metrics_sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" - ``` - - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - **option 3** - _[object]_ - Calls a custom API defined in the project to compute data. - ```yaml - type: api - - api: "user_analytics_api" - args: - start_date: "2024-01-01" - limit: 10 - ``` - - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - **`args`** - _[object]_ - Arguments to pass to the custom API. - **option 4** - _[object]_ - Uses a file-matching pattern (glob) to query data from a connector. - ```yaml - type: api - - glob: "data/*.csv" - ``` - - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ @@ -174,50 +146,22 @@ partitions: - **option 1** - _[object]_ - Executes a raw SQL query against the project's data models. - ```yaml - type: api - - connector: "duckdb" - sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" - ``` - - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. - **option 2** - _[object]_ - Executes a SQL query that targets a defined metrics view. - ```yaml - type: api - - metrics_sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" - ``` - - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - **option 3** - _[object]_ - Calls a custom API defined in the project to compute data. - ```yaml - type: api - - api: "user_analytics_api" - args: - start_date: "2024-01-01" - limit: 10 - ``` - - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - **`args`** - _[object]_ - Arguments to pass to the custom API. - **option 4** - _[object]_ - Uses a file-matching pattern (glob) to query data from a connector. - ```yaml - type: api - - glob: "data/*.csv" - ``` - - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ diff --git a/docs/docs/hidden/yaml/alerts.md b/docs/docs/hidden/yaml/alerts.md index c3152e6fcaf..71bc99e5659 100644 --- a/docs/docs/hidden/yaml/alerts.md +++ b/docs/docs/hidden/yaml/alerts.md @@ -45,50 +45,22 @@ _[string]_ - Description for the alert _[oneOf]_ - Data source for the alert _(required)_ - **option 1** - _[object]_ - Executes a raw SQL query against the project's data models. - ```yaml - type: api - - connector: "duckdb" - sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" - ``` - - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. - **option 2** - _[object]_ - Executes a SQL query that targets a defined metrics view. - ```yaml - type: api - - metrics_sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" - ``` - - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - **option 3** - _[object]_ - Calls a custom API defined in the project to compute data. - ```yaml - type: api - - api: "user_analytics_api" - args: - start_date: "2024-01-01" - limit: 10 - ``` - - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - **`args`** - _[object]_ - Arguments to pass to the custom API. - **option 4** - _[object]_ - Uses a file-matching pattern (glob) to query data from a connector. - ```yaml - type: api - - glob: "data/*.csv" - ``` - - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ diff --git a/runtime/parser/schema/project.schema.yaml b/runtime/parser/schema/project.schema.yaml index 33183edf919..4d5f99dea8a 100644 --- a/runtime/parser/schema/project.schema.yaml +++ b/runtime/parser/schema/project.schema.yaml @@ -1119,7 +1119,7 @@ oneOf: sample: false required: - type - - $ref: '#/definitions/data_properties' + - $ref: '#/definitions/api_data_properties' # - $ref: '#/definitions/common_properties' # Themes @@ -1389,7 +1389,7 @@ definitions: # - name # - type # additionalProperties: false - data_properties: + api_data_properties: oneOf: - title: SQL Query type: object @@ -1476,6 +1476,85 @@ definitions: required: - glob + - title: Resource Status Check + type: object + description: Uses the status of a resource as data. + properties: + resource_status: + type: object + description: Based on resource status + properties: + where_error: + type: boolean + description: Indicates whether the condition should trigger when the resource is in an error state. + sample: true + additionalProperties: true + required: + - resource_status + examples: | + ```yaml + resource_status: + where_error: true + ``` + data_properties: + oneOf: + - title: SQL Query + type: object + description: Executes a raw SQL query against the project's data models. + properties: + sql: + type: string + description: Raw SQL query to run against existing models in the project. + connector: + type: string + description: specifies the connector to use when running SQL or glob queries. + required: + - sql + + + - title: Metrics View Query + type: object + description: Executes a SQL query that targets a defined metrics view. + properties: + metrics_sql: + type: string + description: SQL query that targets a metrics view in the project + required: + - metrics_sql + + - title: Custom API Call + type: object + description: Calls a custom API defined in the project to compute data. + properties: + api: + type: string + description: Name of a custom API defined in the project. + args: + type: object + description: Arguments to pass to the custom API. + additionalProperties: true + required: + - api + + - title: File Glob Query + type: object + description: Uses a file-matching pattern (glob) to query data from a connector. + properties: + glob: + description: Defines the file path or pattern to query from the specified connector. + anyOf: + - type: string + description: A simple file path/glob pattern as a string. + sample: "data/*.csv" + - type: object + description: An object-based configuration for specifying a file path/glob pattern with advanced options. + additionalProperties: true + connector: + type: string + description: Specifies the connector to use with the glob input. + required: + - glob + - title: Resource Status Check type: object description: Uses the status of a resource as data. From e67f1590fd78d7bcd6b66d825c1d3545ca443c4f Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Wed, 6 Aug 2025 22:03:44 -0600 Subject: [PATCH 29/32] gonig through files --- cli/cmd/docs/generate_project.go | 39 +++- docs/docs/hidden/yaml/advanced-models.md | 70 +++--- docs/docs/hidden/yaml/alerts.md | 68 ++---- docs/docs/hidden/yaml/apis.md | 84 ++----- docs/docs/hidden/yaml/canvas-dashboards.md | 41 ++-- docs/docs/hidden/yaml/explore-dashboards.md | 105 +++------ docs/docs/hidden/yaml/metrics-views.md | 28 --- runtime/parser/schema/project.schema.yaml | 232 +++++++++++++------- 8 files changed, 322 insertions(+), 345 deletions(-) diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index faf34438c76..ab063e747e5 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -479,10 +479,9 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req } - - } else { + // only for APIs, no other oneOf uses this, so we can be specific + } else if id == "apis" { // Skip oneOf processing for connectors and data_properties at level 1 since we handle it above - if !(id == "connectors") { if len(oneOf.Content) == 1 { doc.WriteString(generateDoc(sidebarPosition, level, oneOf.Content[0], indent, getRequiredMapFromNode(oneOf.Content[0]), rootSchema, id)) } else { @@ -491,10 +490,10 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req doc.WriteString("\n\n## One of Properties Options") } - for i, item := range oneOf.Content { + for _, item := range oneOf.Content { if hasType(item) || hasProperties(item) || hasCombinators(item) { - doc.WriteString(fmt.Sprintf("\n\n%s- **option %d** - %s - %s", indent, i+1, getPrintableType(item), getPrintableDescription(item, indent, "(no description)"))) doc.WriteString(generateDoc(sidebarPosition, level, item, indent+" ", getRequiredMapFromNode(item), rootSchema, id)) + } // Handle examples for oneOf items @@ -515,6 +514,36 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req } } } + } else { + if !(id == "connectors" || id == "apis") { + // Check if this is a field_selector_properties definition (used in explore dashboards) + title := getScalarValue(node, "title") + if title == "Wildcard(*) selector" || title == "Explicit list of fields" || title == "Regex matching" { + // This is field_selector_properties, use numbered options format + for i, item := range oneOf.Content { + if hasType(item) || hasProperties(item) || hasCombinators(item) { + itemTitle := getScalarValue(item, "title") + if itemTitle != "" { + doc.WriteString(fmt.Sprintf("\n\n%s- **option %d** - _[%s]_ - %s", indent, i+1, getPrintableType(item), itemTitle)) + } else { + doc.WriteString(fmt.Sprintf("\n\n%s- **option %d** - %s - %s", indent, i+1, getPrintableType(item), getPrintableDescription(item, indent, "(no description)"))) + } + doc.WriteString(generateDoc(sidebarPosition, level, item, indent+" ", getRequiredMapFromNode(item), rootSchema, id)) + } + } + } else { + // Regular oneOf processing + if len(oneOf.Content) == 1 { + doc.WriteString(generateDoc(sidebarPosition, level, oneOf.Content[0], indent, getRequiredMapFromNode(oneOf.Content[0]), rootSchema, id)) + } else { + for _, item := range oneOf.Content { + if hasType(item) || hasProperties(item) || hasCombinators(item) { + doc.WriteString(generateDoc(sidebarPosition, level, item, indent+" ", getRequiredMapFromNode(item), rootSchema, id)) + } + } + } + } + } } } } diff --git a/docs/docs/hidden/yaml/advanced-models.md b/docs/docs/hidden/yaml/advanced-models.md index 42b9177c3d6..28e629001ca 100644 --- a/docs/docs/hidden/yaml/advanced-models.md +++ b/docs/docs/hidden/yaml/advanced-models.md @@ -93,24 +93,16 @@ state: ``` - - **option 1** - _[object]_ - Executes a raw SQL query against the project's data models. - - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. - - **option 2** - _[object]_ - Executes a SQL query that targets a defined metrics view. - - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - - **option 3** - _[object]_ - Calls a custom API defined in the project to compute data. - - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - **`args`** - _[object]_ - Arguments to pass to the custom API. - - **option 4** - _[object]_ - Uses a file-matching pattern (glob) to query data from a connector. - - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - **option 1** - _[string]_ - A simple file path/glob pattern as a string. @@ -119,18 +111,10 @@ state: - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. - - **option 5** - _[object]_ - Uses the status of a resource as data. - - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. -```yaml -resource_status: - where_error: true -``` - - ### `partitions` _[oneOf]_ - Refers to the how your data is partitioned, cannot be used with state. (optional) @@ -145,24 +129,16 @@ partitions: ``` - - **option 1** - _[object]_ - Executes a raw SQL query against the project's data models. - - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. - - **option 2** - _[object]_ - Executes a SQL query that targets a defined metrics view. - - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - - **option 3** - _[object]_ - Calls a custom API defined in the project to compute data. - - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - **`args`** - _[object]_ - Arguments to pass to the custom API. - - **option 4** - _[object]_ - Uses a file-matching pattern (glob) to query data from a connector. - - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - **option 1** - _[string]_ - A simple file path/glob pattern as a string. @@ -171,18 +147,10 @@ partitions: - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. - - **option 5** - _[object]_ - Uses the status of a resource as data. - - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. -```yaml -resource_status: - where_error: true -``` - - ### `materialize` _[boolean]_ - models will be materialized in olap @@ -225,6 +193,38 @@ _[object]_ - to define the properties of output - **`partition_by`** - _[string]_ - Column or expression to partition the table by + **Additional properties for `output` when `connector` is `clickhouse`** + + - **`type`** - _[string]_ - Type to materialize the model into. Can be 'TABLE', 'VIEW' or 'DICTIONARY' + + - **`columns`** - _[string]_ - Column names and types. Can also include indexes. If unspecified, detected from the query. + + - **`engine_full`** - _[string]_ - Full engine definition in SQL format. Can include partition keys, order, TTL, etc. + + - **`engine`** - _[string]_ - Table engine to use. Default is MergeTree + + - **`order_by`** - _[string]_ - ORDER BY clause. + + - **`partition_by`** - _[string]_ - Partition BY clause. + + - **`primary_key`** - _[string]_ - PRIMARY KEY clause. + + - **`sample_by`** - _[string]_ - SAMPLE BY clause. + + - **`ttl`** - _[string]_ - TTL settings for the table or columns. + + - **`table_settings`** - _[string]_ - Table-specific settings. + + - **`query_settings`** - _[string]_ - Settings used in insert/create table as select queries. + + - **`distributed_settings`** - _[string]_ - Settings for distributed table. + + - **`distributed_sharding_key`** - _[string]_ - Sharding key for distributed table. + + - **`dictionary_source_user`** - _[string]_ - User for accessing the source dictionary table (used if type is DICTIONARY). + + - **`dictionary_source_password`** - _[string]_ - Password for the dictionary source user. + ## Common Properties ### `name` @@ -241,4 +241,8 @@ _[object]_ - Overrides any properties in development environment. ### `prod` -_[object]_ - Overrides any properties in production environment. \ No newline at end of file +_[object]_ - Overrides any properties in production environment. + +## Depending on the connector, additional properties may be required + +Depending on the connector, additional properties may be required, for more information see the [connectors](./connectors.md) documentation diff --git a/docs/docs/hidden/yaml/alerts.md b/docs/docs/hidden/yaml/alerts.md index 71bc99e5659..8da3f446964 100644 --- a/docs/docs/hidden/yaml/alerts.md +++ b/docs/docs/hidden/yaml/alerts.md @@ -40,28 +40,38 @@ _[string]_ - Display name for the alert _[string]_ - Description for the alert +### `intervals` + +_[object]_ - define the interval of the alert to check + + - **`duration`** - _[string]_ - a valid ISO8601 duration to define the interval duration + + - **`limit`** - _[integer]_ - maximum number of intervals to check for on invocation + + - **`check_unclosed`** - _[boolean]_ - boolean, whether unclosed intervals should be checked + +### `watermark` + +_[string]_ - Specifies how the watermark is determined for incremental processing. Use 'trigger_time' to set it at runtime or 'inherit' to use the upstream model's watermark. + +### `timeout` + +_[string]_ - define the timeout of the alert in seconds (optional). + ### `data` _[oneOf]_ - Data source for the alert _(required)_ - - **option 1** - _[object]_ - Executes a raw SQL query against the project's data models. - - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. - - **option 2** - _[object]_ - Executes a SQL query that targets a defined metrics view. - - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ - - **option 3** - _[object]_ - Calls a custom API defined in the project to compute data. - - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - **`args`** - _[object]_ - Arguments to pass to the custom API. - - **option 4** - _[object]_ - Uses a file-matching pattern (glob) to query data from a connector. - - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - **option 1** - _[string]_ - A simple file path/glob pattern as a string. @@ -70,50 +80,18 @@ _[oneOf]_ - Data source for the alert _(required)_ - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. - - **option 5** - _[object]_ - Uses the status of a resource as data. - - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. -```yaml -resource_status: - where_error: true -``` - - -### `intervals` - -_[object]_ - define the interval of the alert to check - - - **`duration`** - _[string]_ - a valid ISO8601 duration to define the interval duration - - - **`limit`** - _[integer]_ - maximum number of intervals to check for on invocation - - - **`check_unclosed`** - _[boolean]_ - boolean, whether unclosed intervals should be checked - -### `watermark` - -_[string]_ - Specifies how the watermark is determined for incremental processing. Use 'trigger_time' to set it at runtime or 'inherit' to use the upstream model's watermark. - -### `timeout` - -_[string]_ - define the timeout of the alert in seconds (optional). - ### `for` _[oneOf]_ - Specifies how user identity or attributes should be evaluated for security policy enforcement. - - **option 1** - _[object]_ - Specifies a unique user identifier for applying security policies. - - **`user_id`** - _[string]_ - The unique user ID used to evaluate security policies. _(required)_ - - **option 2** - _[object]_ - Specifies a user's email address for applying security policies. - - **`user_email`** - _[string]_ - The user's email address used to evaluate security policies. _(required)_ - - **option 3** - _[object]_ - Specifies a set of arbitrary user attributes for applying security policies. - - **`attributes`** - _[object]_ - A dictionary of user attributes used to evaluate security policies. _(required)_ ### `on_recover` @@ -136,16 +114,6 @@ _[boolean]_ - Enable repeated notifications for unresolved alerts. Defaults to f _[string]_ - Defines the re-notification interval for the alert (e.g., '10m','24h'), equivalent to snooze duration in UI, defaults to 'Off' -### `condition` - -_[object]_ - Condition that triggers the alert - - - **`operator`** - _[string]_ - Comparison operator (gt, lt, eq, etc.) - - - **`threshold`** - _[no type]_ - Threshold value for the condition - - - **`measure`** - _[string]_ - Measure to compare against the threshold - ### `notify` _[object]_ - Notification configuration _(required)_ diff --git a/docs/docs/hidden/yaml/apis.md b/docs/docs/hidden/yaml/apis.md index eb6e6c41db3..5f3006827cb 100644 --- a/docs/docs/hidden/yaml/apis.md +++ b/docs/docs/hidden/yaml/apis.md @@ -30,10 +30,6 @@ _[object]_ - Defines [security rules and access control policies](/manage/securi - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. - - **option 1** - _[string]_ - SQL expression that evaluates to a boolean to determine access - - - **option 2** - _[boolean]_ - Direct boolean value to allow or deny access - - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded @@ -76,25 +72,9 @@ _[boolean]_ - Flag to control security inheritance ## One of Properties Options -- **option 1** - _[object]_ - Executes a raw SQL query against the project's data models. -```yaml -type: api - -connector: "duckdb" -sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" -``` - - ## SQL Query Executes a raw SQL query against the project's data models. - ```yaml - type: api - - connector: "duckdb" - sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" - ``` - ### `sql` @@ -104,51 +84,30 @@ _[string]_ - Raw SQL query to run against existing models in the project. _(requ _[string]_ - specifies the connector to use when running SQL or glob queries. -- **option 2** - _[object]_ - Executes a SQL query that targets a defined metrics view. - ```yaml - type: api - - metrics_sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" - ``` +```yaml +type: api +sql: "SELECT * FROM table_name WHERE date >= '2024-01-01'" +``` ## Metrics View Query Executes a SQL query that targets a defined metrics view. - ```yaml - type: api - - metrics_sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" - ``` - ### `metrics_sql` _[string]_ - SQL query that targets a metrics view in the project _(required)_ -- **option 3** - _[object]_ - Calls a custom API defined in the project to compute data. - ```yaml - type: api +```yaml +type: api - api: "user_analytics_api" - args: - start_date: "2024-01-01" - limit: 10 - ``` +metrics_sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" +``` ## Custom API Call Calls a custom API defined in the project to compute data. - ```yaml - type: api - - api: "user_analytics_api" - args: - start_date: "2024-01-01" - limit: 10 - ``` - ### `api` @@ -158,23 +117,18 @@ _[string]_ - Name of a custom API defined in the project. _(required)_ _[object]_ - Arguments to pass to the custom API. -- **option 4** - _[object]_ - Uses a file-matching pattern (glob) to query data from a connector. - ```yaml - type: api - - glob: "data/*.csv" - ``` +```yaml +type: api +api: "user_analytics_api" +args: + start_date: "2024-01-01" + limit: 10 +``` ## File Glob Query Uses a file-matching pattern (glob) to query data from a connector. - ```yaml - type: api - - glob: "data/*.csv" - ``` - ### `glob` @@ -188,7 +142,12 @@ _[anyOf]_ - Defines the file path or pattern to query from the specified connect _[string]_ - Specifies the connector to use with the glob input. -- **option 5** - _[object]_ - Uses the status of a resource as data. +```yaml +type: api + +glob: "data/*.csv" +``` + ## Resource Status Check @@ -201,6 +160,7 @@ _[object]_ - Based on resource status _(required)_ - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. ```yaml +type: api resource_status: where_error: true ``` diff --git a/docs/docs/hidden/yaml/canvas-dashboards.md b/docs/docs/hidden/yaml/canvas-dashboards.md index f1eb35de023..3d779149c7a 100644 --- a/docs/docs/hidden/yaml/canvas-dashboards.md +++ b/docs/docs/hidden/yaml/canvas-dashboards.md @@ -62,13 +62,11 @@ _[integer]_ - Horizontal gap in pixels of the canvas _[integer]_ - Vertical gap in pixels of the canvas -### `theme` - -_[oneOf]_ - Theme configuration. Can be either a string reference to an existing theme or an inline theme configuration object. +### `filters` - - **option 1** - _[string]_ - Name of an existing theme to apply to the dashboard +_[object]_ - Indicates if filters should be enabled for the canvas. - - **option 2** - _[object]_ - Inline theme configuration. + - **`enable`** - _[boolean]_ - Toggles filtering functionality for the canvas dashboard. ### `allow_custom_time_range` @@ -90,18 +88,10 @@ _[array of oneOf]_ - Overrides the list of default time range selections availab ``` - - **option 1** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection - - - **option 2** - _[object]_ - Object containing time range and comparison configuration - - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) - - **option 1** - _[string]_ - Offset string only (range is inferred) - - - **option 2** - _[object]_ - Object containing offset and range configuration for time comparison - - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) - **`range`** - _[string]_ - Custom time range for comparison period @@ -110,9 +100,26 @@ _[array of oneOf]_ - Overrides the list of default time range selections availab _[array of string]_ - Refers to the time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) -### `filters` +### `defaults` -_[object]_ - Indicates if filters should be enabled for the canvas. +_[object]_ - defines the defaults YAML struct + ```yaml + defaults: #define all the defaults within here + time_range: P1M + comparison_mode: dimension #time, none + comparison_dimension: filename + ``` + + + - **`time_range`** - _[string]_ - Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + + - **`comparison_mode`** - _[string]_ - Controls how to compare current data with historical or categorical baselines. Options: `none` (no comparison), `time` (compares with past based on default_time_range), `dimension` (compares based on comparison_dimension values) + + - **`comparison_dimension`** - _[string]_ - for dimension mode, specify the comparison dimension by name + +### `theme` + +_[oneOf]_ - Theme configuration. Can be either a string reference to an existing theme or an inline theme configuration object. ### `security` @@ -120,10 +127,6 @@ _[object]_ - Defines [security rules and access control policies](/manage/securi - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. - - **option 1** - _[string]_ - SQL expression that evaluates to a boolean to determine access - - - **option 2** - _[boolean]_ - Direct boolean value to allow or deny access - ## Common Properties ### `name` diff --git a/docs/docs/hidden/yaml/explore-dashboards.md b/docs/docs/hidden/yaml/explore-dashboards.md index 352f64671ed..a0c953e1baf 100644 --- a/docs/docs/hidden/yaml/explore-dashboards.md +++ b/docs/docs/hidden/yaml/explore-dashboards.md @@ -30,28 +30,7 @@ _[string]_ - Refers to the custom banner displayed at the header of an explore d ### `dimensions` -_[oneOf]_ - List of dimension names. Use '*' to select all dimensions (default) - ```yaml - # Example: Select a dimension - dimensions: - - country - - # Example: Select all dimensions except one - dimensions: - exclude: - - country - - # Example: Select all dimensions that match a regex - dimensions: - regex: "^public_.*$" - ``` - - - - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection - - - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection - - - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion +_[oneOf]_ - List of dimension names. Use '*' to select all dimensions (default) - **`regex`** - _[string]_ - Select dimensions using a regular expression @@ -59,15 +38,25 @@ _[oneOf]_ - List of dimension names. Use '*' to select all dimensions (default) - **`exclude`** - _[object]_ - Select all dimensions except those listed here -### `measures` +```yaml +# Example: Select a dimension +dimensions: + - country -_[oneOf]_ - List of measure names. Use '*' to select all measures (default) +# Example: Select all dimensions except one +dimensions: + exclude: + - country - - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection +# Example: Select all dimensions that match a regex +dimensions: +regex: "^public_.*$" +``` - - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection - - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion +### `measures` + +_[oneOf]_ - List of measure names. Use '*' to select all measures (default) - **`regex`** - _[string]_ - Select dimensions using a regular expression @@ -75,33 +64,25 @@ _[oneOf]_ - List of measure names. Use '*' to select all measures (default) - **`exclude`** - _[object]_ - Select all dimensions except those listed here -### `theme` - -_[oneOf]_ - Name of the theme to use. Only one of theme and embedded_theme can be set. - - - **option 1** - _[string]_ - Name of an existing theme to apply to the dashboard - - - **option 2** - _[object]_ - Inline theme configuration. - -### `time_ranges` - -_[oneOf]_ - Default time range for the dashboard - - - **option 1** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection - - - **option 2** - _[object]_ - Object containing time range and comparison configuration +```yaml +# Example: Select a dimension +measures: + - sum_of_total - - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ - - - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) +# Example: Select all dimensions except one +measures: + exclude: + - sum_of_total - - **option 1** - _[string]_ - Offset string only (range is inferred) +# Example: Select all dimensions that match a regex +measures: +regex: "^public_.*$" +``` - - **option 2** - _[object]_ - Object containing offset and range configuration for time comparison - - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) +### `theme` - - **`range`** - _[string]_ - Custom time range for comparison period +_[oneOf]_ - Name of the theme to use. Only one of theme and embedded_theme can be set. ### `time_ranges` @@ -119,18 +100,10 @@ _[array of oneOf]_ - Overrides the list of default time range selections availab ``` - - **option 1** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection - - - **option 2** - _[object]_ - Object containing time range and comparison configuration - - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) - - **option 1** - _[string]_ - Offset string only (range is inferred) - - - **option 2** - _[object]_ - Object containing offset and range configuration for time comparison - - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) - **`range`** - _[string]_ - Custom time range for comparison period @@ -166,12 +139,6 @@ _[object]_ - defines the defaults YAML struct - **`dimensions`** - _[oneOf]_ - Provides the default dimensions to load on viewing the dashboard - - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection - - - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection - - - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion - - **`regex`** - _[string]_ - Select dimensions using a regular expression - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic @@ -180,19 +147,13 @@ _[object]_ - defines the defaults YAML struct - **`measures`** - _[oneOf]_ - Provides the default measures to load on viewing the dashboard - - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection - - - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection - - - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion - - **`regex`** - _[string]_ - Select dimensions using a regular expression - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - **`exclude`** - _[object]_ - Select all dimensions except those listed here - - **`time_ranges`** - _[string]_ - Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + - **`time_range`** - _[string]_ - Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) - **`comparison_mode`** - _[string]_ - Controls how to compare current data with historical or categorical baselines. Options: `none` (no comparison), `time` (compares with past based on default_time_range), `dimension` (compares based on comparison_dimension values) @@ -210,10 +171,6 @@ _[object]_ - Defines [security rules and access control policies](/manage/securi - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. - - **option 1** - _[string]_ - SQL expression that evaluates to a boolean to determine access - - - **option 2** - _[boolean]_ - Direct boolean value to allow or deny access - ## Common Properties ### `name` diff --git a/docs/docs/hidden/yaml/metrics-views.md b/docs/docs/hidden/yaml/metrics-views.md index c9a7807a259..bb237de541c 100644 --- a/docs/docs/hidden/yaml/metrics-views.md +++ b/docs/docs/hidden/yaml/metrics-views.md @@ -112,14 +112,6 @@ _[array of object]_ - Used to define the numeric aggregates of columns from your - **`order`** - _[string]_ - Specifies the fields to order the window by, determining the sequence of rows within each partition. - - **option 1** - _[string]_ - Simple field name as a string. - - - **option 2** - _[array of oneOf]_ - List of field selectors, each can be a string or an object with detailed configuration. - - - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. - - - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. - - **`name`** - _[string]_ - Name of the field to select. _(required)_ - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. @@ -128,28 +120,12 @@ _[array of object]_ - Used to define the numeric aggregates of columns from your - **`per`** - _[oneOf]_ - for per dimensions - - **option 1** - _[string]_ - Simple field name as a string. - - - **option 2** - _[array of oneOf]_ - List of field selectors, each can be a string or an object with detailed configuration. - - - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. - - - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. - - **`name`** - _[string]_ - Name of the field to select. _(required)_ - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. - **`requires`** - _[oneOf]_ - using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures. See [referencing measures](/build/metrics-view/advanced-expressions/referencing) for more information. - - **option 1** - _[string]_ - Simple field name as a string. - - - **option 2** - _[array of oneOf]_ - List of field selectors, each can be a string or an object with detailed configuration. - - - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. - - - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. - - **`name`** - _[string]_ - Name of the field to select. _(required)_ - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. @@ -220,10 +196,6 @@ _[object]_ - Defines [security rules and access control policies](/manage/securi - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. - - **option 1** - _[string]_ - SQL expression that evaluates to a boolean to determine access - - - **option 2** - _[boolean]_ - Direct boolean value to allow or deny access - - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded diff --git a/runtime/parser/schema/project.schema.yaml b/runtime/parser/schema/project.schema.yaml index 4d5f99dea8a..3bff36c8bd4 100644 --- a/runtime/parser/schema/project.schema.yaml +++ b/runtime/parser/schema/project.schema.yaml @@ -400,10 +400,72 @@ oneOf: partition_by: type: string description: Column or expression to partition the table by + allOf: + - if: + title: Additional properties for `output` when `connector` is `clickhouse` + properties: + connector: + const: clickhouse + required: + - connector + then: + properties: + type: + type: string + description: Type to materialize the model into. Can be 'TABLE', 'VIEW' or 'DICTIONARY' + enum: + - TABLE + - VIEW + - DICTIONARY + columns: + type: string + description: Column names and types. Can also include indexes. If unspecified, detected from the query. + engine_full: + type: string + description: Full engine definition in SQL format. Can include partition keys, order, TTL, etc. + engine: + type: string + description: Table engine to use. Default is MergeTree + order_by: + type: string + description: ORDER BY clause. + partition_by: + type: string + description: Partition BY clause. + primary_key: + type: string + description: PRIMARY KEY clause. + sample_by: + type: string + description: SAMPLE BY clause. + ttl: + type: string + description: TTL settings for the table or columns. + table_settings: + type: string + description: Table-specific settings. + query_settings: + type: string + description: Settings used in insert/create table as select queries. + distributed_settings: + type: string + description: Settings for distributed table. + distributed_sharding_key: + type: string + description: Sharding key for distributed table. + dictionary_source_user: + type: string + description: User for accessing the source dictionary table (used if type is DICTIONARY). + dictionary_source_password: + type: string + description: Password for the dictionary source user. required: - type - sql - $ref: '#/definitions/common_properties' + - title: Depending on the connector, additional properties may be required + description: | + Depending on the connector, additional properties may be required, for more information see the [connectors](./connectors.md) documentation # Metrics Views - title: Metrics View YAML @@ -728,7 +790,6 @@ oneOf: description: Width of the component (can be a number or string with unit) additionalProperties: true additionalProperties: false - max_width: type: integer description: Max width in pixels of the canvas @@ -741,15 +802,14 @@ oneOf: type: integer description: Vertical gap in pixels of the canvas minimum: 0 - theme: - oneOf: - - title: Existing theme - type: string - description: Name of an existing theme to apply to the dashboard - - title: Inline theme - type: object - description: Inline theme configuration. - description: Theme configuration. Can be either a string reference to an existing theme or an inline theme configuration object. + filters: + type: object + description: Indicates if filters should be enabled for the canvas. + additionalProperties: true + properties: + enable: + type: boolean + description: Toggles filtering functionality for the canvas dashboard. allow_custom_time_range: type: boolean description: Defaults to true, when set to false it will hide the ability to set a custom time range for the user. @@ -775,10 +835,41 @@ oneOf: description: Refers to the time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) items: type: string - filters: + defaults: type: object - description: Indicates if filters should be enabled for the canvas. - additionalProperties: true + description: | + defines the defaults YAML struct + ```yaml + defaults: #define all the defaults within here + time_range: P1M + comparison_mode: dimension #time, none + comparison_dimension: filename + ``` + properties: + time_range: + description: Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + type: string + comparison_mode: + description: 'Controls how to compare current data with historical or categorical baselines. Options: `none` (no comparison), `time` (compares with past based on default_time_range), `dimension` (compares based on comparison_dimension values)' + type: string + enum: + - none + - time + - dimension + comparison_dimension: + description: 'for dimension mode, specify the comparison dimension by name' + type: string + additionalProperties: false + theme: + oneOf: + - title: Existing theme + type: string + description: Name of an existing theme to apply to the dashboard + - title: Inline theme + type: object + description: Inline theme configuration. + description: Theme configuration. Can be either a string reference to an existing theme or an inline theme configuration object. + security: description: Security rules to apply for access to the canvas dashboard $ref: '#/definitions/dashboard_security_policy_properties' @@ -813,8 +904,9 @@ oneOf: type: string description: Refers to the custom banner displayed at the header of an explore dashboard dimensions: - description: | - List of dimension names. Use '*' to select all dimensions (default) + description: List of dimension names. Use '*' to select all dimensions (default) + $ref: '#/definitions/field_selector_properties' + examples: | ```yaml # Example: Select a dimension dimensions: @@ -829,10 +921,24 @@ oneOf: dimensions: regex: "^public_.*$" ``` - $ref: '#/definitions/field_selector_properties' measures: description: List of measure names. Use '*' to select all measures (default) $ref: '#/definitions/field_selector_properties' + examples: | + ```yaml + # Example: Select a dimension + measures: + - sum_of_total + + # Example: Select all dimensions except one + measures: + exclude: + - sum_of_total + + # Example: Select all dimensions that match a regex + measures: + regex: "^public_.*$" + ``` theme: oneOf: - title: Existing theme @@ -842,9 +948,6 @@ oneOf: type: object description: Inline theme configuration. description: Name of the theme to use. Only one of theme and embedded_theme can be set. - time_ranges: - $ref: '#/definitions/explore_time_range_properties' - description: Default time range for the dashboard time_ranges: type: array description: | @@ -896,12 +999,16 @@ oneOf: measures: description: Provides the default measures to load on viewing the dashboard $ref: '#/definitions/field_selector_properties' - time_ranges: + time_range: description: Refers to the default time range shown when a user initially loads the dashboard. The value must be either a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) (for example, PT12H for 12 hours, P1M for 1 month, or P26W for 26 weeks) or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) type: string comparison_mode: description: 'Controls how to compare current data with historical or categorical baselines. Options: `none` (no comparison), `time` (compares with past based on default_time_range), `dimension` (compares based on comparison_dimension values)' type: string + enum: + - none + - time + - dimension comparison_dimension: description: 'for dimension mode, specify the comparison dimension by name' type: string @@ -974,9 +1081,6 @@ oneOf: description: type: string description: Description for the alert - data: - $ref: '#/definitions/data_properties' - description: Data source for the alert intervals: type: object description: define the interval of the alert to check @@ -1000,6 +1104,9 @@ oneOf: timeout: type: string description: define the timeout of the alert in seconds (optional). + data: + $ref: '#/definitions/data_properties' + description: Data source for the alert for: description: "Specifies how user identity or attributes should be evaluated for security policy enforcement." oneOf: @@ -1047,18 +1154,6 @@ oneOf: renotify_after: type: string description: Defines the re-notification interval for the alert (e.g., '10m','24h'), equivalent to snooze duration in UI, defaults to 'Off' - condition: - type: object - description: Condition that triggers the alert - properties: - operator: - type: string - description: Comparison operator (gt, lt, eq, etc.) - threshold: - description: Threshold value for the condition - measure: - type: string - description: Measure to compare against the threshold notify: $ref: '#/definitions/notify_properties' description: Notification configuration @@ -1393,14 +1488,8 @@ definitions: oneOf: - title: SQL Query type: object - description: | - Executes a raw SQL query against the project's data models. - ```yaml - type: api + description: Executes a raw SQL query against the project's data models. - connector: "duckdb" - sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" - ``` properties: sql: type: string @@ -1410,36 +1499,29 @@ definitions: description: specifies the connector to use when running SQL or glob queries. required: - sql - - + examples: | + ```yaml + type: api + sql: "SELECT * FROM table_name WHERE date >= '2024-01-01'" + ``` - title: Metrics View Query type: object - description: | - Executes a SQL query that targets a defined metrics view. - ```yaml - type: api - - metrics_sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" - ``` + description: Executes a SQL query that targets a defined metrics view. properties: metrics_sql: type: string description: SQL query that targets a metrics view in the project required: - metrics_sql + examples: | + ```yaml + type: api + metrics_sql: "SELECT * FROM user_metrics WHERE date >= '2024-01-01'" + ``` - title: Custom API Call type: object - description: | - Calls a custom API defined in the project to compute data. - ```yaml - type: api - - api: "user_analytics_api" - args: - start_date: "2024-01-01" - limit: 10 - ``` + description: Calls a custom API defined in the project to compute data. properties: api: type: string @@ -1450,16 +1532,17 @@ definitions: additionalProperties: true required: - api - + examples: | + ```yaml + type: api + api: "user_analytics_api" + args: + start_date: "2024-01-01" + limit: 10 + ``` - title: File Glob Query type: object - description: | - Uses a file-matching pattern (glob) to query data from a connector. - ```yaml - type: api - - glob: "data/*.csv" - ``` + description: Uses a file-matching pattern (glob) to query data from a connector. properties: glob: description: Defines the file path or pattern to query from the specified connector. @@ -1475,7 +1558,12 @@ definitions: description: Specifies the connector to use with the glob input. required: - glob + examples: | + ```yaml + type: api + glob: "data/*.csv" + ``` - title: Resource Status Check type: object description: Uses the status of a resource as data. @@ -1493,6 +1581,7 @@ definitions: - resource_status examples: | ```yaml + type: api resource_status: where_error: true ``` @@ -1510,8 +1599,6 @@ definitions: description: specifies the connector to use when running SQL or glob queries. required: - sql - - - title: Metrics View Query type: object description: Executes a SQL query that targets a defined metrics view. @@ -1521,7 +1608,6 @@ definitions: description: SQL query that targets a metrics view in the project required: - metrics_sql - - title: Custom API Call type: object description: Calls a custom API defined in the project to compute data. @@ -1535,7 +1621,6 @@ definitions: additionalProperties: true required: - api - - title: File Glob Query type: object description: Uses a file-matching pattern (glob) to query data from a connector. @@ -1554,7 +1639,6 @@ definitions: description: Specifies the connector to use with the glob input. required: - glob - - title: Resource Status Check type: object description: Uses the status of a resource as data. From df54d71a4591f413d85a0368f2006c7813c83317 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Thu, 7 Aug 2025 10:16:47 -0600 Subject: [PATCH 30/32] returning original format next will be adding more examples --- cli/cmd/docs/generate_project.go | 33 +- docs/docs/hidden/yaml/advanced-models.md | 88 ++++-- docs/docs/hidden/yaml/alerts.md | 16 + docs/docs/hidden/yaml/canvas-dashboards.md | 24 +- docs/docs/hidden/yaml/explore-dashboards.md | 46 +++ docs/docs/hidden/yaml/metrics-views.md | 28 ++ docs/docs/hidden/yaml/themes.md | 13 +- runtime/parser/schema/project.schema.yaml | 318 ++++++++++++-------- 8 files changed, 381 insertions(+), 185 deletions(-) diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index ab063e747e5..9b3b876624f 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -516,31 +516,16 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req } } else { if !(id == "connectors" || id == "apis") { - // Check if this is a field_selector_properties definition (used in explore dashboards) - title := getScalarValue(node, "title") - if title == "Wildcard(*) selector" || title == "Explicit list of fields" || title == "Regex matching" { - // This is field_selector_properties, use numbered options format - for i, item := range oneOf.Content { - if hasType(item) || hasProperties(item) || hasCombinators(item) { - itemTitle := getScalarValue(item, "title") - if itemTitle != "" { - doc.WriteString(fmt.Sprintf("\n\n%s- **option %d** - _[%s]_ - %s", indent, i+1, getPrintableType(item), itemTitle)) - } else { - doc.WriteString(fmt.Sprintf("\n\n%s- **option %d** - %s - %s", indent, i+1, getPrintableType(item), getPrintableDescription(item, indent, "(no description)"))) } - doc.WriteString(generateDoc(sidebarPosition, level, item, indent+" ", getRequiredMapFromNode(item), rootSchema, id)) - } - } - } else { - // Regular oneOf processing - if len(oneOf.Content) == 1 { - doc.WriteString(generateDoc(sidebarPosition, level, oneOf.Content[0], indent, getRequiredMapFromNode(oneOf.Content[0]), rootSchema, id)) - } else { - for _, item := range oneOf.Content { - if hasType(item) || hasProperties(item) || hasCombinators(item) { - doc.WriteString(generateDoc(sidebarPosition, level, item, indent+" ", getRequiredMapFromNode(item), rootSchema, id)) - } - } + + if len(oneOf.Content) == 1 { + doc.WriteString(generateDoc(sidebarPosition, level, oneOf.Content[0], indent, getRequiredMapFromNode(oneOf.Content[0]), rootSchema, id)) + } else { + for i, item := range oneOf.Content { + if hasType(item) || hasProperties(item) || hasCombinators(item) { + doc.WriteString(fmt.Sprintf("\n\n%s- **option %d** - %s - %s", indent, i+1, getPrintableType(item), getPrintableDescription(item, indent, "(no description)"))) + doc.WriteString(generateDoc(sidebarPosition, level, item, indent+" ", getRequiredMapFromNode(item), rootSchema, id)) + } } } diff --git a/docs/docs/hidden/yaml/advanced-models.md b/docs/docs/hidden/yaml/advanced-models.md index 28e629001ca..04df304f789 100644 --- a/docs/docs/hidden/yaml/advanced-models.md +++ b/docs/docs/hidden/yaml/advanced-models.md @@ -29,12 +29,6 @@ _[string]_ - Refers to the resource type and must be `model` _(required)_ ### `refresh` _[object]_ - Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying model data -```yaml -refresh: - cron: "* * * * *" - #every: "24h" -``` - - **`cron`** - _[string]_ - A cron expression that defines the execution schedule @@ -46,6 +40,13 @@ refresh: - **`run_in_dev`** - _[boolean]_ - If true, allows the schedule to run in development mode. +```yaml +refresh: + cron: "* * * * *" + #every: "24h" +``` + + ### `connector` _[string]_ - Refers to the resource type and is needed if setting an explicit OLAP engine. IE `clickhouse` @@ -86,23 +87,26 @@ _[string]_ - Configure how changes to the model specifications are applied (opti ### `state` -_[oneOf]_ - Refers to the explicitly defined state of your model, cannot be used with partitions (optional) -```yaml -state: - sql: SELECT MAX(date) as max_date -``` - +_[oneOf]_ - Refers to the explicitly defined state of your model, cannot be used with partitions (optional) + + - **option 1** - _[object]_ - Executes a raw SQL query against the project's data models. - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. + - **option 2** - _[object]_ - Executes a SQL query that targets a defined metrics view. + - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ + - **option 3** - _[object]_ - Calls a custom API defined in the project to compute data. + - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - **`args`** - _[object]_ - Arguments to pass to the custom API. + - **option 4** - _[object]_ - Uses a file-matching pattern (glob) to query data from a connector. + - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - **option 1** - _[string]_ - A simple file path/glob pattern as a string. @@ -111,34 +115,40 @@ state: - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. + - **option 5** - _[object]_ - Uses the status of a resource as data. + - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. -### `partitions` - -_[oneOf]_ - Refers to the how your data is partitioned, cannot be used with state. (optional) ```yaml -partitions: - glob: gcs://my_bucket/y=*/m=*/d=*/*.parquet +state: + sql: SELECT MAX(date) as max_date ``` -```yaml -partitions: - connector: duckdb - sql: SELECT range AS num FROM range(0,10) - ``` - + + +### `partitions` + +_[oneOf]_ - Refers to the how your data is partitioned, cannot be used with state. (optional) + + - **option 1** - _[object]_ - Executes a raw SQL query against the project's data models. - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. + - **option 2** - _[object]_ - Executes a SQL query that targets a defined metrics view. + - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ + - **option 3** - _[object]_ - Calls a custom API defined in the project to compute data. + - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - **`args`** - _[object]_ - Arguments to pass to the custom API. + - **option 4** - _[object]_ - Uses a file-matching pattern (glob) to query data from a connector. + - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - **option 1** - _[string]_ - A simple file path/glob pattern as a string. @@ -147,10 +157,23 @@ partitions: - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. + - **option 5** - _[object]_ - Uses the status of a resource as data. + - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. +```yaml +partitions: + glob: gcs://my_bucket/y=*/m=*/d=*/*.parquet +``` +```yaml +partitions: + connector: duckdb + sql: SELECT range AS num FROM range(0,10) + ``` + + ### `materialize` _[boolean]_ - models will be materialized in olap @@ -165,17 +188,18 @@ _[integer]_ - Refers to the number of concurrent partitions that can be read at ### `stage` -_[object]_ - in the case of staging models, where an input source does not support direct write to the output and a staging table is required +_[object]_ - in the case of staging models, where an input source does not support direct write to the output and a staging table is required + + - **`connector`** - _[string]_ - Refers to the connector type for the staging table _(required)_ + + - **`path`** - _[string]_ - Refers to the path to the staging table + ```yaml stage: connector: s3 path: s3://my_bucket/my_staging_table ``` - - - **`connector`** - _[string]_ - Refers to the connector type for the staging table _(required)_ - - - **`path`** - _[string]_ - Refers to the path to the staging table ### `output` @@ -246,3 +270,11 @@ _[object]_ - Overrides any properties in production environment. ## Depending on the connector, additional properties may be required Depending on the connector, additional properties may be required, for more information see the [connectors](./connectors.md) documentation + + +## Examples + +### Incremental model +```yaml +test +``` diff --git a/docs/docs/hidden/yaml/alerts.md b/docs/docs/hidden/yaml/alerts.md index 8da3f446964..2a93f42ffd5 100644 --- a/docs/docs/hidden/yaml/alerts.md +++ b/docs/docs/hidden/yaml/alerts.md @@ -62,16 +62,24 @@ _[string]_ - define the timeout of the alert in seconds (optional). _[oneOf]_ - Data source for the alert _(required)_ + - **option 1** - _[object]_ - Executes a raw SQL query against the project's data models. + - **`sql`** - _[string]_ - Raw SQL query to run against existing models in the project. _(required)_ - **`connector`** - _[string]_ - specifies the connector to use when running SQL or glob queries. + - **option 2** - _[object]_ - Executes a SQL query that targets a defined metrics view. + - **`metrics_sql`** - _[string]_ - SQL query that targets a metrics view in the project _(required)_ + - **option 3** - _[object]_ - Calls a custom API defined in the project to compute data. + - **`api`** - _[string]_ - Name of a custom API defined in the project. _(required)_ - **`args`** - _[object]_ - Arguments to pass to the custom API. + - **option 4** - _[object]_ - Uses a file-matching pattern (glob) to query data from a connector. + - **`glob`** - _[anyOf]_ - Defines the file path or pattern to query from the specified connector. _(required)_ - **option 1** - _[string]_ - A simple file path/glob pattern as a string. @@ -80,6 +88,8 @@ _[oneOf]_ - Data source for the alert _(required)_ - **`connector`** - _[string]_ - Specifies the connector to use with the glob input. + - **option 5** - _[object]_ - Uses the status of a resource as data. + - **`resource_status`** - _[object]_ - Based on resource status _(required)_ - **`where_error`** - _[boolean]_ - Indicates whether the condition should trigger when the resource is in an error state. @@ -88,10 +98,16 @@ _[oneOf]_ - Data source for the alert _(required)_ _[oneOf]_ - Specifies how user identity or attributes should be evaluated for security policy enforcement. + - **option 1** - _[object]_ - Specifies a unique user identifier for applying security policies. + - **`user_id`** - _[string]_ - The unique user ID used to evaluate security policies. _(required)_ + - **option 2** - _[object]_ - Specifies a user's email address for applying security policies. + - **`user_email`** - _[string]_ - The user's email address used to evaluate security policies. _(required)_ + - **option 3** - _[object]_ - Specifies a set of arbitrary user attributes for applying security policies. + - **`attributes`** - _[object]_ - A dictionary of user attributes used to evaluate security policies. _(required)_ ### `on_recover` diff --git a/docs/docs/hidden/yaml/canvas-dashboards.md b/docs/docs/hidden/yaml/canvas-dashboards.md index 3d779149c7a..327136a5485 100644 --- a/docs/docs/hidden/yaml/canvas-dashboards.md +++ b/docs/docs/hidden/yaml/canvas-dashboards.md @@ -88,10 +88,18 @@ _[array of oneOf]_ - Overrides the list of default time range selections availab ``` + - **option 1** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection + + - **option 2** - _[object]_ - Object containing time range and comparison configuration + - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + - **option 1** - _[string]_ - Offset string only (range is inferred) + + - **option 2** - _[object]_ - Object containing offset and range configuration for time comparison + - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) - **`range`** - _[string]_ - Custom time range for comparison period @@ -119,7 +127,17 @@ _[object]_ - defines the defaults YAML struct ### `theme` -_[oneOf]_ - Theme configuration. Can be either a string reference to an existing theme or an inline theme configuration object. +_[oneOf]_ - Name of the theme to use. Only one of theme and embedded_theme can be set. + + - **option 1** - _[string]_ - Name of an existing theme to apply to the dashboard + + - **option 2** - _[object]_ - Inline theme configuration. + + - **`colors`** - _[object]_ - Used to override the dashboard colors. Either primary or secondary color must be provided. + + - **`primary`** - _[string]_ - Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). + + - **`secondary`** - _[string]_ - Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. ### `security` @@ -127,6 +145,10 @@ _[object]_ - Defines [security rules and access control policies](/manage/securi - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + - **option 1** - _[string]_ - SQL expression that evaluates to a boolean to determine access + + - **option 2** - _[boolean]_ - Direct boolean value to allow or deny access + ## Common Properties ### `name` diff --git a/docs/docs/hidden/yaml/explore-dashboards.md b/docs/docs/hidden/yaml/explore-dashboards.md index a0c953e1baf..a2d474f438e 100644 --- a/docs/docs/hidden/yaml/explore-dashboards.md +++ b/docs/docs/hidden/yaml/explore-dashboards.md @@ -32,6 +32,12 @@ _[string]_ - Refers to the custom banner displayed at the header of an explore d _[oneOf]_ - List of dimension names. Use '*' to select all dimensions (default) + - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection + + - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection + + - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion + - **`regex`** - _[string]_ - Select dimensions using a regular expression - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic @@ -58,6 +64,12 @@ regex: "^public_.*$" _[oneOf]_ - List of measure names. Use '*' to select all measures (default) + - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection + + - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection + + - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion + - **`regex`** - _[string]_ - Select dimensions using a regular expression - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic @@ -84,6 +96,16 @@ regex: "^public_.*$" _[oneOf]_ - Name of the theme to use. Only one of theme and embedded_theme can be set. + - **option 1** - _[string]_ - Name of an existing theme to apply to the dashboard + + - **option 2** - _[object]_ - Inline theme configuration. + + - **`colors`** - _[object]_ - Used to override the dashboard colors. Either primary or secondary color must be provided. + + - **`primary`** - _[string]_ - Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). + + - **`secondary`** - _[string]_ - Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. + ### `time_ranges` _[array of oneOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets' @@ -100,10 +122,18 @@ _[array of oneOf]_ - Overrides the list of default time range selections availab ``` + - **option 1** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection + + - **option 2** - _[object]_ - Object containing time range and comparison configuration + - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) + - **option 1** - _[string]_ - Offset string only (range is inferred) + + - **option 2** - _[object]_ - Object containing offset and range configuration for time comparison + - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) - **`range`** - _[string]_ - Custom time range for comparison period @@ -139,6 +169,12 @@ _[object]_ - defines the defaults YAML struct - **`dimensions`** - _[oneOf]_ - Provides the default dimensions to load on viewing the dashboard + - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection + + - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection + + - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion + - **`regex`** - _[string]_ - Select dimensions using a regular expression - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic @@ -147,6 +183,12 @@ _[object]_ - defines the defaults YAML struct - **`measures`** - _[oneOf]_ - Provides the default measures to load on viewing the dashboard + - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection + + - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection + + - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion + - **`regex`** - _[string]_ - Select dimensions using a regular expression - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic @@ -171,6 +213,10 @@ _[object]_ - Defines [security rules and access control policies](/manage/securi - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + - **option 1** - _[string]_ - SQL expression that evaluates to a boolean to determine access + + - **option 2** - _[boolean]_ - Direct boolean value to allow or deny access + ## Common Properties ### `name` diff --git a/docs/docs/hidden/yaml/metrics-views.md b/docs/docs/hidden/yaml/metrics-views.md index bb237de541c..c9a7807a259 100644 --- a/docs/docs/hidden/yaml/metrics-views.md +++ b/docs/docs/hidden/yaml/metrics-views.md @@ -112,6 +112,14 @@ _[array of object]_ - Used to define the numeric aggregates of columns from your - **`order`** - _[string]_ - Specifies the fields to order the window by, determining the sequence of rows within each partition. + - **option 1** - _[string]_ - Simple field name as a string. + + - **option 2** - _[array of oneOf]_ - List of field selectors, each can be a string or an object with detailed configuration. + + - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. + + - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. + - **`name`** - _[string]_ - Name of the field to select. _(required)_ - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. @@ -120,12 +128,28 @@ _[array of object]_ - Used to define the numeric aggregates of columns from your - **`per`** - _[oneOf]_ - for per dimensions + - **option 1** - _[string]_ - Simple field name as a string. + + - **option 2** - _[array of oneOf]_ - List of field selectors, each can be a string or an object with detailed configuration. + + - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. + + - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. + - **`name`** - _[string]_ - Name of the field to select. _(required)_ - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. - **`requires`** - _[oneOf]_ - using an available measure or dimension in your metrics view to set a required parameter, cannot be used with simple measures. See [referencing measures](/build/metrics-view/advanced-expressions/referencing) for more information. + - **option 1** - _[string]_ - Simple field name as a string. + + - **option 2** - _[array of oneOf]_ - List of field selectors, each can be a string or an object with detailed configuration. + + - **option 1** - _[string]_ - Shorthand field selector, interpreted as the name. + + - **option 2** - _[object]_ - Detailed field selector configuration with name and optional time grain. + - **`name`** - _[string]_ - Name of the field to select. _(required)_ - **`time_grain`** - _[string]_ - Time grain for time-based dimensions. @@ -196,6 +220,10 @@ _[object]_ - Defines [security rules and access control policies](/manage/securi - **`access`** - _[oneOf]_ - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + - **option 1** - _[string]_ - SQL expression that evaluates to a boolean to determine access + + - **option 2** - _[boolean]_ - Direct boolean value to allow or deny access + - **`row_filter`** - _[string]_ - SQL expression to filter the underlying model by. Can leverage templated user attributes to customize the filter for the requesting user. Needs to be a valid SQL expression that can be injected into a WHERE clause - **`include`** - _[array of object]_ - List of dimension or measure names to include in the dashboard. If include is defined all other dimensions and measures are excluded diff --git a/docs/docs/hidden/yaml/themes.md b/docs/docs/hidden/yaml/themes.md index 3186e7e929d..5f8c88d4ef5 100644 --- a/docs/docs/hidden/yaml/themes.md +++ b/docs/docs/hidden/yaml/themes.md @@ -39,4 +39,15 @@ _[object]_ - Overrides any properties in development environment. ### `prod` -_[object]_ - Overrides any properties in production environment. \ No newline at end of file +_[object]_ - Overrides any properties in production environment. + +## Examples + +```yaml +# Example: You can copy this directly into your .yaml file +type: theme + +colors: + primary: plum + secondary: violet +``` diff --git a/runtime/parser/schema/project.schema.yaml b/runtime/parser/schema/project.schema.yaml index 3bff36c8bd4..bd4e0560f06 100644 --- a/runtime/parser/schema/project.schema.yaml +++ b/runtime/parser/schema/project.schema.yaml @@ -21,8 +21,22 @@ description: | ::: oneOf: + - $ref: '#/definitions/connectors' + - $ref: '#/definitions/sources' + - $ref: '#/definitions/models' + - $ref: '#/definitions/advanced-models' + - $ref: '#/definitions/metrics-views' + - $ref: '#/definitions/canvas-dashboards' + - $ref: '#/definitions/explore-dashboards' + - $ref: '#/definitions/alerts' + - $ref: '#/definitions/apis' + - $ref: '#/definitions/themes' + - $ref: '#/definitions/rillyaml' + +definitions: # Connector YAML - - title: Connector YAML + connectors: + title: Connector YAML id: connectors type: object description: | @@ -94,9 +108,10 @@ oneOf: - $ref: '#/definitions/slack' - $ref: '#/definitions/snowflake' - $ref: '#/definitions/sqlite' - + # Source YAML - - title: Source YAML + sources: + title: Source YAML type: object id: sources description: | @@ -197,8 +212,10 @@ oneOf: - type - connector - $ref: '#/definitions/common_properties' + # Model SQL - - title: Model SQL + models: + title: Model SQL type: object id: models description: | @@ -250,7 +267,9 @@ oneOf: ::: # Advanced Models - - title: Models YAML + advanced-models: + + title: Models YAML id: advanced-models type: object description: | @@ -278,8 +297,8 @@ oneOf: description: Refers to the resource type and must be `model` refresh: $ref: '#/definitions/schedule_properties' - description: | - Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying model data + description: Specifies the refresh schedule that Rill should follow to re-ingest and update the underlying model data + examples: | ```yaml refresh: cron: "* * * * *" @@ -324,16 +343,16 @@ oneOf: description: Configure how changes to the model specifications are applied (optional). 'reset' will drop and recreate the model automatically, 'manual' will require a manual full or incremental refresh to apply changes, and 'patch' will switch to the new logic without re-processing historical data (only applies for incremental models). state: $ref: '#/definitions/data_properties' - description: | - Refers to the explicitly defined state of your model, cannot be used with partitions (optional) + description: Refers to the explicitly defined state of your model, cannot be used with partitions (optional) + examples: | ```yaml state: sql: SELECT MAX(date) as max_date ``` partitions: $ref: '#/definitions/data_properties' - description: | - Refers to the how your data is partitioned, cannot be used with state. (optional) + description: Refers to the how your data is partitioned, cannot be used with state. (optional) + examples: | ```yaml partitions: glob: gcs://my_bucket/y=*/m=*/d=*/*.parquet @@ -364,8 +383,8 @@ oneOf: required: - connector - description: | - in the case of staging models, where an input source does not support direct write to the output and a staging table is required + description: in the case of staging models, where an input source does not support direct write to the output and a staging table is required + examples: | ```yaml stage: connector: s3 @@ -466,9 +485,15 @@ oneOf: - title: Depending on the connector, additional properties may be required description: | Depending on the connector, additional properties may be required, for more information see the [connectors](./connectors.md) documentation - + examples: | + ### Incremental model + ```yaml + test + ``` + # Metrics Views - - title: Metrics View YAML + metrics-views: + title: Metrics View YAML id: metrics-views type: object description: In your Rill project directory, create a metrics view, `.yaml`, file in the `metrics` directory. Rill will ingest the metric view definition next time you run `rill start`. @@ -730,7 +755,8 @@ oneOf: - $ref: '#/definitions/common_properties' # Canvas Dashboards - - title: Canvas Dashboard YAML + canvas-dashboards: + title: Canvas Dashboard YAML id: canvas-dashboards type: object description: Canvas dashboards provide a flexible way to create custom dashboards with drag-and-drop components. @@ -862,14 +888,11 @@ oneOf: additionalProperties: false theme: oneOf: - - title: Existing theme - type: string + - type: string description: Name of an existing theme to apply to the dashboard - - title: Inline theme - type: object + - $ref: '#/definitions/theme_properties' description: Inline theme configuration. - description: Theme configuration. Can be either a string reference to an existing theme or an inline theme configuration object. - + description: Name of the theme to use. Only one of theme and embedded_theme can be set. security: description: Security rules to apply for access to the canvas dashboard $ref: '#/definitions/dashboard_security_policy_properties' @@ -878,8 +901,9 @@ oneOf: - display_name - $ref: '#/definitions/common_properties' - # Explore Dashboards - - title: Explore Dashboard YAML + # Explore dashboards + explore-dashboards: + title: Explore Dashboard YAML id: explore-dashboards type: object description: Explore dashboards provide an interactive way to explore data with predefined metrics and dimensions. @@ -941,11 +965,9 @@ oneOf: ``` theme: oneOf: - - title: Existing theme - type: string + - type: string description: Name of an existing theme to apply to the dashboard - - title: Inline theme - type: object + - $ref: '#/definitions/theme_properties' description: Inline theme configuration. description: Name of the theme to use. Only one of theme and embedded_theme can be set. time_ranges: @@ -1032,7 +1054,8 @@ oneOf: - $ref: '#/definitions/common_properties' # Alerts - - title: Alert YAML + alerts: + title: Alert YAML id: alerts type: object description: Along with alertings at the dashboard level and can be created via the UI, there might be more extensive alerting that you might want to develop and can be done so the an alert.yaml. When creating an alert via a YAML file, you'll see this denoted in the UI as `Created through code`. @@ -1171,7 +1194,8 @@ oneOf: - $ref: '#/definitions/common_properties' # APIs - - title: API YAML + apis: + title: API YAML id: apis type: object description: Custom APIs allow you to create endpoints that can be called to retrieve or manipulate data. @@ -1218,14 +1242,23 @@ oneOf: # - $ref: '#/definitions/common_properties' # Themes - - title: Theme YAML + themes: + title: Theme YAML id: themes type: object description: | In your Rill project directory, create a `.yaml` file in any directory containing `type: theme`. Rill will automatically ingest the theme next time you run `rill start` or deploy to Rill Cloud. To apply that theme to a dashboard, add `default_theme: ` to the yaml file for that dashboard. Alternatively, you can add this to the end of the URL in your browser: `?theme=` - + examples: | + ```yaml + # Example: You can copy this directly into your .yaml file + type: theme + + colors: + primary: plum + secondary: violet + ``` allOf: - title: Properties type: object @@ -1252,7 +1285,8 @@ oneOf: - $ref: '#/definitions/common_properties' # Rill YAML - - title: Project YAML + rillyaml: + title: Project YAML id: rillyaml type: object description: The `rill.yaml` file contains metadata about your project. @@ -1443,7 +1477,8 @@ oneOf: required: - email - $ref: '#/definitions/common_properties' -definitions: + + # Common Properties common_properties: type: object title: "Common Properties" @@ -1463,27 +1498,26 @@ definitions: prod: type: object description: Overrides any properties in production environment. - # component_variable_properties: - # type: object - # properties: - # name: - # type: string - # description: Unique identifier for the variable - # type: - # type: string - # description: Data type of the variable (e.g., string, number, boolean) - # value: - # description: Default value for the variable. Can be any valid JSON value type - # type: - # - string - # - number - # - boolean - # - object - # - array - # required: - # - name - # - type - # additionalProperties: false + + # Reusable Properties + theme_properties: + type: object + properties: + colors: + type: object + description: Used to override the dashboard colors. Either primary or secondary color must be provided. + properties: + primary: + type: string + description: Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). + secondary: + type: string + description: Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. + anyOf: + - required: + - primary + - required: + - secondary api_data_properties: oneOf: - title: SQL Query @@ -1852,7 +1886,6 @@ definitions: description: SQL expression for row filtering (for row_filter type rules) required: - type - dashboard_security_policy_properties: type: object description: Defines [security rules and access control policies](/manage/security) for dashboards (without row filtering) @@ -1864,8 +1897,81 @@ definitions: - type: boolean description: Direct boolean value to allow or deny access description: Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to false and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean. + field_selectors_properties: + oneOf: + - type: string + description: 'Simple field name as a string.' + - type: array + description: 'List of field selectors, each can be a string or an object with detailed configuration.' + items: + oneOf: + - type: string + description: 'Shorthand field selector, interpreted as the name.' + - type: object + description: 'Detailed field selector configuration with name and optional time grain.' + properties: + name: + type: string + description: 'Name of the field to select.' + time_grain: + type: string + description: 'Time grain for time-based dimensions.' + enum: + - '' + - ms + - millisecond + - s + - second + - min + - minute + - h + - hour + - d + - day + - w + - week + - month + - q + - quarter + - 'y' + - year + required: + - name + additionalProperties: false + minItems: 1 + field_selector_properties: + oneOf: + - title: Wildcard(*) selector + type: string + const: '*' + description: Wildcard(*) selector that includes all available fields in the selection + - title: Explicit list of fields + type: array + items: + type: string + description: Explicit list of fields to include in the selection + - title: Regex matching + type: object + description: 'Advanced matching using regex, DuckDB expression, or exclusion' + properties: + regex: + type: string + description: Select dimensions using a regular expression + expr: + type: string + description: DuckDB SQL expression to select fields based on custom logic + exclude: + type: object + description: Select all dimensions except those listed here + additionalProperties: false + oneOf: + - required: + - regex + - required: + - expr + - required: + - exclude - # Connector definitions athena: type: object @@ -2525,77 +2631,27 @@ definitions: - dsn - driver - dsn - field_selectors_properties: - oneOf: - - type: string - description: 'Simple field name as a string.' - - type: array - description: 'List of field selectors, each can be a string or an object with detailed configuration.' - items: - oneOf: - - type: string - description: 'Shorthand field selector, interpreted as the name.' - - type: object - description: 'Detailed field selector configuration with name and optional time grain.' - properties: - name: - type: string - description: 'Name of the field to select.' - time_grain: - type: string - description: 'Time grain for time-based dimensions.' - enum: - - '' - - ms - - millisecond - - s - - second - - min - - minute - - h - - hour - - d - - day - - w - - week - - month - - q - - quarter - - 'y' - - year - required: - - name - additionalProperties: false - minItems: 1 - field_selector_properties: - oneOf: - - title: Wildcard(*) selector - type: string - const: '*' - description: Wildcard(*) selector that includes all available fields in the selection - - title: Explicit list of fields - type: array - items: - type: string - description: Explicit list of fields to include in the selection - - title: Regex matching - type: object - description: 'Advanced matching using regex, DuckDB expression, or exclusion' - properties: - regex: - type: string - description: Select dimensions using a regular expression - expr: - type: string - description: DuckDB SQL expression to select fields based on custom logic - exclude: - type: object - description: Select all dimensions except those listed here - additionalProperties: false - oneOf: - - required: - - regex - - required: - - expr - - required: - - exclude \ No newline at end of file + + + + # component_variable_properties: + # type: object + # properties: + # name: + # type: string + # description: Unique identifier for the variable + # type: + # type: string + # description: Data type of the variable (e.g., string, number, boolean) + # value: + # description: Default value for the variable. Can be any valid JSON value type + # type: + # - string + # - number + # - boolean + # - object + # - array + # required: + # - name + # - type + # additionalProperties: false \ No newline at end of file From f94396efb8a78a088820c152f175076413c5222e Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Thu, 7 Aug 2025 10:57:02 -0600 Subject: [PATCH 31/32] ? --- cli/cmd/docs/generate_project.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/cmd/docs/generate_project.go b/cli/cmd/docs/generate_project.go index 9b3b876624f..22501221e69 100644 --- a/cli/cmd/docs/generate_project.go +++ b/cli/cmd/docs/generate_project.go @@ -516,7 +516,6 @@ func generateDoc(sidebarPosition, level int, node *yaml.Node, indent string, req } } else { if !(id == "connectors" || id == "apis") { - } if len(oneOf.Content) == 1 { doc.WriteString(generateDoc(sidebarPosition, level, oneOf.Content[0], indent, getRequiredMapFromNode(oneOf.Content[0]), rootSchema, id)) From e8c04e94dda58da26230a415043c27d89265ed49 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 8 Aug 2025 11:20:11 -0600 Subject: [PATCH 32/32] fixing urls, --- docs/docs/hidden/yaml/connectors.md | 2 +- docs/docs/hidden/yaml/explore-dashboards.md | 2 +- docs/docs/hidden/yaml/metrics-views.md | 126 +------------------- docs/docs/hidden/yaml/rillyaml.md | 8 +- runtime/parser/schema/project.schema.yaml | 12 +- 5 files changed, 14 insertions(+), 136 deletions(-) diff --git a/docs/docs/hidden/yaml/connectors.md b/docs/docs/hidden/yaml/connectors.md index d5800bfade1..fe19cf56686 100644 --- a/docs/docs/hidden/yaml/connectors.md +++ b/docs/docs/hidden/yaml/connectors.md @@ -37,7 +37,7 @@ Connector YAML files define how Rill connects to external data sources and OLAP - [**Slack**](#slack) - Slack data :::warning Security Recommendation -For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env.connector..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/build/credentials/) for complete setup instructions. +For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env.connector..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/connect/credentials/) for complete setup instructions. ::: diff --git a/docs/docs/hidden/yaml/explore-dashboards.md b/docs/docs/hidden/yaml/explore-dashboards.md index 2305a22cde5..3dab0169ad4 100644 --- a/docs/docs/hidden/yaml/explore-dashboards.md +++ b/docs/docs/hidden/yaml/explore-dashboards.md @@ -94,7 +94,7 @@ regex: "^public_.*$" ### `theme` -_[oneOf]_ - Name of the theme to use or define a theme inline. Either theme name or inline theme can be set. +_[oneOf]_ - Name of the theme to use. Only one of theme and embedded_theme can be set. - **option 1** - _[string]_ - Name of an existing theme to apply to the dashboard diff --git a/docs/docs/hidden/yaml/metrics-views.md b/docs/docs/hidden/yaml/metrics-views.md index 5c1b8ffc1b4..d9674622940 100644 --- a/docs/docs/hidden/yaml/metrics-views.md +++ b/docs/docs/hidden/yaml/metrics-views.md @@ -18,7 +18,7 @@ _[string]_ - Refers to the resource type and must be `metrics_view` _(required)_ ### `connector` -_[string]_ - Refers to the connector type for the metrics view, see [OLAP engines](/build/olap) for more information +_[string]_ - Refers to the connector type for the metrics view, see [OLAP engines](/connect/olap) for more information ### `display_name` @@ -184,45 +184,13 @@ _[array of object]_ - Used to define the numeric aggregates of columns from your - **`treat_nulls_as`** - _[string]_ - used to configure what value to fill in for missing time buckets. This also works generally as COALESCING over non empty time buckets. -### `parent_dimensions` - -_[oneOf]_ - Optional field selectors for dimensions to inherit from the parent metrics view. - - - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection - - - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection - - - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion - - - **`regex`** - _[string]_ - Select fields using a regular expression - - - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - - - **`exclude`** - _[object]_ - Select all fields except those listed here - -### `parent_measures` - -_[oneOf]_ - Optional field selectors for measures to inherit from the parent metrics view. - - - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection - - - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection - - - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion - - - **`regex`** - _[string]_ - Select fields using a regular expression - - - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - - - **`exclude`** - _[object]_ - Select all fields except those listed here - ### `annotations` _[array of object]_ - Used to define annotations that can be displayed on charts - **`name`** - _[string]_ - A stable identifier for the annotation. Defaults to model or table names when not specified - - **`model`** - _[string]_ - Refers to the model powering the annotation (either table or model is required). The model must have 'time' and 'description' columns. Optional columns include 'time_end' for range annotations and 'duration' to specify when the annotation should appear based on dashboard grain level. + - **`model`** - _[string]_ - Refers to the model powering the annotation (either table or model is required). The model must have 'time' and 'description' columns. Optional columns include 'time_end' for range annotations and 'grain' to specify when the annotation should appear based on dashboard grain level. - **`database`** - _[string]_ - Refers to the database to use in the OLAP engine (to be used in conjunction with table). Otherwise, will use the default database or schema if not specified @@ -292,96 +260,6 @@ _[object]_ - Defines [security rules and access control policies](/manage/securi - **`sql`** - _[string]_ - SQL expression for row filtering (for row_filter type rules) -### `explore` - -_[object]_ - Defines an optional inline explore view for the metrics view. If not specified a default explore will be emitted unless `skip` is set to true. - - - **`skip`** - _[boolean]_ - If true, disables the explore view for this metrics view. - - - **`name`** - _[string]_ - Name of the explore view. - - - **`display_name`** - _[string]_ - Display name for the explore view. - - - **`description`** - _[string]_ - Description for the explore view. - - - **`banner`** - _[string]_ - Custom banner displayed at the header of the explore view. - - - **`theme`** - _[oneOf]_ - Name of the theme to use or define a theme inline. Either theme name or inline theme can be set. - - - **option 1** - _[string]_ - Name of an existing theme to apply to the explore view. - - - **option 2** - _[object]_ - Inline theme configuration. - - - **`colors`** - _[object]_ - Used to override the dashboard colors. Either primary or secondary color must be provided. - - - **`primary`** - _[string]_ - Overrides the primary blue color in the dashboard. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. Note that the hue of the input colors is used for variants but the saturation and lightness is copied over from the [blue color palette](https://tailwindcss.com/docs/customizing-colors). - - - **`secondary`** - _[string]_ - Overrides the secondary color in the dashboard. Applies to the loading spinner only as of now. Can have any hex (without the '#' character), [named colors](https://www.w3.org/TR/css-color-4/#named-colors) or hsl() formats. - - - **`time_ranges`** - _[array of oneOf]_ - Overrides the list of default time range selections available in the dropdown. It can be string or an object with a 'range' and optional 'comparison_offsets'. - - - **option 1** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection - - - **option 2** - _[object]_ - Object containing time range and comparison configuration - - - **`range`** - _[string]_ - a valid [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) duration or one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) extensions for the selection _(required)_ - - - **`comparison_offsets`** - _[array of oneOf]_ - list of time comparison options for this time range selection (optional). Must be one of the [Rill ISO 8601 extensions](https://docs.rilldata.com/reference/rill-iso-extensions#extensions) - - - **option 1** - _[string]_ - Offset string only (range is inferred) - - - **option 2** - _[object]_ - Object containing offset and range configuration for time comparison - - - **`offset`** - _[string]_ - Time offset for comparison (e.g., 'P1D' for one day ago) - - - **`range`** - _[string]_ - Custom time range for comparison period - - - **`time_zones`** - _[array of string]_ - List of time zones to pin to the top of the time zone selector. Should be a list of IANA time zone identifiers. - - - **`lock_time_zone`** - _[boolean]_ - When true, the explore view will be locked to the first time zone provided in the time_zones list. If no time_zones are provided, it will be locked to UTC. - - - **`allow_custom_time_range`** - _[boolean]_ - Defaults to true. When set to false, hides the ability to set a custom time range for the user. - - - **`defaults`** - _[object]_ - Preset UI state to show by default. - - - **`dimensions`** - _[oneOf]_ - Default dimensions to load on viewing the explore view. - - - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection - - - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection - - - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion - - - **`regex`** - _[string]_ - Select fields using a regular expression - - - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - - - **`exclude`** - _[object]_ - Select all fields except those listed here - - - **`measures`** - _[oneOf]_ - Default measures to load on viewing the explore view. - - - **option 1** - _[string]_ - Wildcard(*) selector that includes all available fields in the selection - - - **option 2** - _[array of string]_ - Explicit list of fields to include in the selection - - - **option 3** - _[object]_ - Advanced matching using regex, DuckDB expression, or exclusion - - - **`regex`** - _[string]_ - Select fields using a regular expression - - - **`expr`** - _[string]_ - DuckDB SQL expression to select fields based on custom logic - - - **`exclude`** - _[object]_ - Select all fields except those listed here - - - **`time_range`** - _[string]_ - Default time range to display when the explore view loads. - - - **`comparison_mode`** - _[string]_ - Default comparison mode for metrics (none, time, or dimension). - - - **`comparison_dimension`** - _[string]_ - Default dimension to use for comparison when comparison_mode is 'dimension'. - - - **`embeds`** - _[object]_ - Configuration options for embedded explore views. - - - **`hide_pivot`** - _[boolean]_ - When true, hides the pivot table view in embedded mode. - ## Common Properties ### `name` diff --git a/docs/docs/hidden/yaml/rillyaml.md b/docs/docs/hidden/yaml/rillyaml.md index 97d5c9d46cf..d70073805c2 100644 --- a/docs/docs/hidden/yaml/rillyaml.md +++ b/docs/docs/hidden/yaml/rillyaml.md @@ -32,13 +32,13 @@ _[string]_ - Extra instructions for [AI agents](/explore/mcp). Used to guide nat Rill allows you to specify the default OLAP engine to use in your project via `rill.yaml`. :::info Curious about OLAP Engines? -Please see our reference documentation on [OLAP Engines](/connect). +Please see our reference documentation on [OLAP Engines](/connect/olap). ::: ### `olap_connector` -_[string]_ - Specifies the [default OLAP engine](/build/olap) for the project. Defaults to duckdb if not set. +_[string]_ - Specifies the [default OLAP engine](/connect/olap) for the project. Defaults to duckdb if not set. ```yaml olap_connector: clickhouse @@ -74,7 +74,7 @@ _[object]_ - Defines project-wide default settings for explores. Unless overridd ```yaml # For example, the following YAML configuration below will set a project-wide default for: -# Models - Configure a [source refresh](/build/models/source-refresh). +# Models - Configure a [source refresh](/build/connect/source-refresh.md). # Metrics View - Set the [first day of the week](metrics-view.md) for timeseries aggregations to be Sunday along with setting the smallest_time_grain. # Explore Dashboards - Set the [default](explore-dashboards.md) values when a user opens a dashboard, and available time zones and/or time ranges. models: @@ -112,7 +112,7 @@ explores: ## Setting variables -Primarily useful for [templating](/connect/templating), variables can be set in the `rill.yaml` file directly. This allows variables to be set for your projects deployed to Rill Cloud while still being able to use different variable values locally if you prefer. +Primarily useful for [templating](/connect/templating.md), variables can be set in the `rill.yaml` file directly. This allows variables to be set for your projects deployed to Rill Cloud while still being able to use different variable values locally if you prefer. :::info Overriding variables locally Variables also follow an order of precedence and can be overridden locally. By default, any variables defined will be inherited from `rill.yaml`. However, if you manually pass in a variable when starting Rill Developer locally via the CLI, this value will be used instead for the current instance of your running project: ```bash diff --git a/runtime/parser/schema/project.schema.yaml b/runtime/parser/schema/project.schema.yaml index accb58efed3..8cfc4c8bf51 100644 --- a/runtime/parser/schema/project.schema.yaml +++ b/runtime/parser/schema/project.schema.yaml @@ -73,7 +73,7 @@ definitions: - [**Slack**](#slack) - Slack data :::warning Security Recommendation - For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env.connector..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/build/credentials/) for complete setup instructions. + For all credential parameters (passwords, tokens, keys), use environment variables with the syntax `{{.env.connector..}}`. This keeps sensitive data out of your YAML files and version control. See our [credentials documentation](/connect/credentials/) for complete setup instructions. ::: allOf: - title: Properties @@ -510,7 +510,7 @@ definitions: description: Refers to the resource type and must be `metrics_view` connector: type: string - description: Refers to the connector type for the metrics view, see [OLAP engines](/build/olap) for more information + description: Refers to the connector type for the metrics view, see [OLAP engines](/connect/olap) for more information display_name: type: string description: Refers to the display name for the metrics view @@ -1313,13 +1313,13 @@ definitions: description: | Rill allows you to specify the default OLAP engine to use in your project via `rill.yaml`. :::info Curious about OLAP Engines? - Please see our reference documentation on [OLAP Engines](/docs/reference/olap-engines/olap-engines.md). + Please see our reference documentation on [OLAP Engines](/connect/olap). ::: type: object properties: olap_connector: type: string - description: Specifies the [default OLAP engine](/build/olap) for the project. Defaults to duckdb if not set. + description: Specifies the [default OLAP engine](/connect/olap) for the project. Defaults to duckdb if not set. examples: - olap_connector: clickhouse - title: Project-wide defaults @@ -1387,7 +1387,7 @@ definitions: - P12M - title: Setting variables description: | - Primarily useful for [templating](/deploy/templating.md), variables can be set in the `rill.yaml` file directly. This allows variables to be set for your projects deployed to Rill Cloud while still being able to use different variable values locally if you prefer. + Primarily useful for [templating](/connect/templating.md), variables can be set in the `rill.yaml` file directly. This allows variables to be set for your projects deployed to Rill Cloud while still being able to use different variable values locally if you prefer. :::info Overriding variables locally Variables also follow an order of precedence and can be overridden locally. By default, any variables defined will be inherited from `rill.yaml`. However, if you manually pass in a variable when starting Rill Developer locally via the CLI, this value will be used instead for the current instance of your running project: ```bash @@ -1399,7 +1399,7 @@ definitions: ```bash variable=xyz ``` - Similar to how [connector credentials can be pushed / pulled](/build/credentials/credentials.md#pulling-credentials-and-variables-from-a-deployed-project-on-rill-cloud) from local to cloud or vice versa, project variables set locally in Rill Developer can be pushed to Rill Cloud and/or pulled back to your local instance from your deployed project by using the `rill env push` and `rill env pull` commands respectively. + Similar to how [connector credentials can be pushed / pulled](/connect/credentials#pulling-credentials-and-variables-from-a-deployed-project-on-rill-cloud) from local to cloud or vice versa, project variables set locally in Rill Developer can be pushed to Rill Cloud and/or pulled back to your local instance from your deployed project by using the `rill env push` and `rill env pull` commands respectively. ::: type: object properties: