From 50df07c742ef8b78dfa8ebfa0628509b9b4bf713 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Mon, 4 Aug 2025 18:52:08 -0700 Subject: [PATCH 01/21] create postgres connector init --- runtime/drivers/postgres/postgres.go | 4 ++-- .../sources/modal/submitAddDataForm.ts | 19 ++++++++++------- .../src/features/sources/modal/yupSchemas.ts | 4 ++-- .../src/features/sources/sourceUtils.ts | 21 +++++++------------ 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/runtime/drivers/postgres/postgres.go b/runtime/drivers/postgres/postgres.go index dbb5f599fe5..dd7ddb91aa7 100644 --- a/runtime/drivers/postgres/postgres.go +++ b/runtime/drivers/postgres/postgres.go @@ -38,7 +38,7 @@ var spec = drivers.Spec{ { Key: "sql", Type: drivers.StringPropertyType, - Required: true, + Required: false, DisplayName: "SQL", Description: "Query to extract data from Postgres.", Placeholder: "select * from table;", @@ -59,7 +59,7 @@ var spec = drivers.Spec{ DisplayName: "Source name", Description: "The name of the source", Placeholder: "my_new_source", - Required: true, + Required: false, }, }, ImplementsSQLStore: true, diff --git a/web-common/src/features/sources/modal/submitAddDataForm.ts b/web-common/src/features/sources/modal/submitAddDataForm.ts index 87ea3447120..5f7f81acb56 100644 --- a/web-common/src/features/sources/modal/submitAddDataForm.ts +++ b/web-common/src/features/sources/modal/submitAddDataForm.ts @@ -51,13 +51,18 @@ export async function submitAddSourceForm( formValues, ); - // Make a new .yaml file - const newSourceFilePath = getFileAPIPathFromNameAndType( - formValues.name as string, - EntityType.Table, + const newConnectorName = getName( + connector.name as string, + fileArtifacts.getNamesForKind(ResourceKind.Connector), + ); + + // Make a new .yaml file + const newConnectorFilePath = getFileAPIPathFromNameAndType( + newConnectorName, + EntityType.Connector, ); await runtimeServicePutFile(instanceId, { - path: newSourceFilePath, + path: newConnectorFilePath, blob: compileSourceYAML(rewrittenConnector, rewrittenFormValues), create: true, createOnly: false, // The modal might be opened from a YAML file with placeholder text, so the file might already exist @@ -70,13 +75,13 @@ export async function submitAddSourceForm( queryClient, rewrittenConnector, rewrittenFormValues, - "source", + "connector", ), create: true, createOnly: false, }); - await goto(`/files/${newSourceFilePath}`); + await goto(`/files/${newConnectorFilePath}`); } export async function submitAddOLAPConnectorForm( diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 4ae2ac6ccd4..f78dd07cda6 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -92,12 +92,12 @@ export const getYupSchema = { }), postgres: yup.object().shape({ - sql: yup.string().required("sql is required"), + sql: yup.string().optional(), database_url: yup.string(), name: yup .string() .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), + .optional(), }), snowflake: yup.object().shape({ diff --git a/web-common/src/features/sources/sourceUtils.ts b/web-common/src/features/sources/sourceUtils.ts index 598959e8e15..d2dd7d56ee5 100644 --- a/web-common/src/features/sources/sourceUtils.ts +++ b/web-common/src/features/sources/sourceUtils.ts @@ -9,11 +9,10 @@ import { makeDotEnvConnectorKey } from "../connectors/code-utils"; import { sanitizeEntityName } from "../entity-management/name-utils"; // Helper text that we put at the top of every Source YAML file -const TOP_OF_FILE = `# Source YAML -# Reference documentation: https://docs.rilldata.com/reference/project-files/sources +const TOP_OF_FILE = `# Connector YAML +# Reference documentation: https://docs.rilldata.com/reference/project-files/connectors -type: model -materialize: true`; +type: connector`; export function compileSourceYAML( connector: V1ConnectorDriver, @@ -36,6 +35,7 @@ export function compileSourceYAML( // Compile key value pairs const compiledKeyValues = Object.keys(formValues) .filter((key) => { + // For connector files, exclude user-provided name since we use connector type if (key === "name") return false; const value = formValues[key]; if (value === undefined) return false; @@ -48,10 +48,7 @@ export function compileSourceYAML( const isSecretProperty = secretPropertyKeys.includes(key); if (isSecretProperty) { - // In Source YAML, explictly referencing `.env` secrets is not yet supported - // For now, `.env` secrets are implicitly referenced - return; - + // For connector files, we include secret properties return `${key}: "{{ .env.${makeDotEnvConnectorKey( connector.name as string, key, @@ -76,15 +73,11 @@ export function compileSourceYAML( .join("\n"); // Return the compiled YAML - return ( - `${TOP_OF_FILE}\n\nconnector: "${connector.name}"\n` + compiledKeyValues - ); + return `${TOP_OF_FILE}\n\ndriver: ${connector.name}\n` + compiledKeyValues; } export function compileLocalFileSourceYAML(path: string) { - return `${TOP_OF_FILE}\n\nconnector: "duckdb"\nsql: "${buildDuckDbQuery( - path, - )}"`; + return `${TOP_OF_FILE}\n\ndriver: duckdb\nsql: "${buildDuckDbQuery(path)}"`; } function buildDuckDbQuery(path: string): string { From c232bfc18aa63397d714871e29830786d2f6f521 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 5 Aug 2025 13:26:55 -0700 Subject: [PATCH 02/21] remove sql and name from postgres driver --- runtime/drivers/postgres/postgres.go | 18 +----------------- .../src/features/sources/modal/yupSchemas.ts | 7 +------ 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/runtime/drivers/postgres/postgres.go b/runtime/drivers/postgres/postgres.go index dd7ddb91aa7..3fb223075e9 100644 --- a/runtime/drivers/postgres/postgres.go +++ b/runtime/drivers/postgres/postgres.go @@ -35,32 +35,16 @@ var spec = drivers.Spec{ }, // Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas. SourceProperties: []*drivers.PropertySpec{ - { - Key: "sql", - Type: drivers.StringPropertyType, - Required: false, - DisplayName: "SQL", - Description: "Query to extract data from Postgres.", - Placeholder: "select * from table;", - }, { Key: "database_url", Type: drivers.StringPropertyType, DisplayName: "Postgres Connection String", - Required: false, + Required: true, DocsURL: "https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING", Placeholder: "postgresql://postgres:postgres@localhost:5432/postgres", Hint: "Can be configured here or by setting the 'connector.postgres.database_url' environment variable (using '.env' or '--env')", Secret: true, }, - { - Key: "name", - Type: drivers.StringPropertyType, - DisplayName: "Source name", - Description: "The name of the source", - Placeholder: "my_new_source", - Required: false, - }, }, ImplementsSQLStore: true, } diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index f78dd07cda6..6f3bfd19686 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -92,12 +92,7 @@ export const getYupSchema = { }), postgres: yup.object().shape({ - sql: yup.string().optional(), - database_url: yup.string(), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .optional(), + database_url: yup.string().required("Database URL is required"), }), snowflake: yup.object().shape({ From 31f94ea10b7b67edcf94a6eaeff0a7456879fb9d Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 5 Aug 2025 14:21:24 -0700 Subject: [PATCH 03/21] remove sql and name from mysql --- runtime/drivers/mysql/mysql.go | 18 +----------------- .../src/features/sources/modal/yupSchemas.ts | 7 +------ 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/runtime/drivers/mysql/mysql.go b/runtime/drivers/mysql/mysql.go index 97211ff7e4a..1095ee5fa92 100644 --- a/runtime/drivers/mysql/mysql.go +++ b/runtime/drivers/mysql/mysql.go @@ -36,32 +36,16 @@ var spec = drivers.Spec{ }, // Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas. SourceProperties: []*drivers.PropertySpec{ - { - Key: "sql", - Type: drivers.StringPropertyType, - Required: true, - DisplayName: "SQL", - Description: "Query to extract data from MySQL.", - Placeholder: "select * from table;", - }, { Key: "dsn", Type: drivers.StringPropertyType, DisplayName: "MySQL Connection String", - Required: false, + Required: true, DocsURL: "https://dev.mysql.com/doc/refman/8.4/en/connecting-using-uri-or-key-value-pairs.html#connecting-using-uri", Placeholder: "mysql://user:password@host:3306/my-db", Hint: "Can be configured here or by setting the 'connector.mysql.dsn' environment variable (using '.env' or '--env')", Secret: true, }, - { - Key: "name", - Type: drivers.StringPropertyType, - DisplayName: "Source name", - Description: "The name of the source", - Placeholder: "my_new_source", - Required: true, - }, }, ImplementsSQLStore: true, } diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 6f3bfd19686..a05ff1b585f 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -143,12 +143,7 @@ export const getYupSchema = { }), mysql: yup.object().shape({ - sql: yup.string().required("sql is required"), - dsn: yup.string(), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), + dsn: yup.string().required("DSN is required"), }), clickhouse: yup.object().shape({ From 709f29b9c6e76a1e4f804f24148ec2c50a719da4 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 5 Aug 2025 16:42:35 -0700 Subject: [PATCH 04/21] remove name from gcs driver --- runtime/drivers/gcs/gcs.go | 8 -------- web-common/src/features/sources/modal/yupSchemas.ts | 4 ---- 2 files changed, 12 deletions(-) diff --git a/runtime/drivers/gcs/gcs.go b/runtime/drivers/gcs/gcs.go index 9da60510ff5..1b42a6a4e97 100644 --- a/runtime/drivers/gcs/gcs.go +++ b/runtime/drivers/gcs/gcs.go @@ -44,14 +44,6 @@ var spec = drivers.Spec{ Required: true, Hint: "Glob patterns are supported", }, - { - Key: "name", - Type: drivers.StringPropertyType, - DisplayName: "Source name", - Description: "The name of the source", - Placeholder: "my_new_source", - Required: true, - }, }, ImplementsObjectStore: true, } diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index a05ff1b585f..ca6f1901dec 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -22,10 +22,6 @@ export const getYupSchema = { .string() .matches(/^gs:\/\//, "Must be a GS URI (e.g. gs://bucket/path)") .required("GS URI is required"), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), }), https: yup.object().shape({ From bf0f5776d853fe5d255e01778644e9c22fa7ea70 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 5 Aug 2025 16:45:06 -0700 Subject: [PATCH 05/21] remove s3 from rewrite to duckdb --- web-common/src/features/sources/sourceUtils.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/web-common/src/features/sources/sourceUtils.ts b/web-common/src/features/sources/sourceUtils.ts index d2dd7d56ee5..96c6321b204 100644 --- a/web-common/src/features/sources/sourceUtils.ts +++ b/web-common/src/features/sources/sourceUtils.ts @@ -158,7 +158,6 @@ export function maybeRewriteToDuckDb( switch (connector.name) { case "s3": - case "gcs": case "https": case "azure": case "local_file": From f3b55f71b4325f952bf3696909359f2f9a93a442 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 6 Aug 2025 11:52:24 -0700 Subject: [PATCH 06/21] Revert "remove s3 from rewrite to duckdb" This reverts commit bf0f5776d853fe5d255e01778644e9c22fa7ea70. --- web-common/src/features/sources/sourceUtils.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/web-common/src/features/sources/sourceUtils.ts b/web-common/src/features/sources/sourceUtils.ts index 96c6321b204..d2dd7d56ee5 100644 --- a/web-common/src/features/sources/sourceUtils.ts +++ b/web-common/src/features/sources/sourceUtils.ts @@ -158,6 +158,7 @@ export function maybeRewriteToDuckDb( switch (connector.name) { case "s3": + case "gcs": case "https": case "azure": case "local_file": From 3bf311cce648a914b96edb95b8045ecf25b0e1f8 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 6 Aug 2025 16:26:36 -0700 Subject: [PATCH 07/21] set source imported path for connector case --- .../entity-management/WatchResourcesClient.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/web-common/src/features/entity-management/WatchResourcesClient.ts b/web-common/src/features/entity-management/WatchResourcesClient.ts index 796528cbc6e..acafade957c 100644 --- a/web-common/src/features/entity-management/WatchResourcesClient.ts +++ b/web-common/src/features/entity-management/WatchResourcesClient.ts @@ -120,6 +120,7 @@ export class WatchResourcesClient { switch (res.name.kind as ResourceKind) { case ResourceKind.Connector: + console.log("connector: ", res.resource); // Invalidate the list of connectors void queryClient.invalidateQueries({ queryKey: getRuntimeServiceAnalyzeConnectorsQueryKey( @@ -135,11 +136,27 @@ export class WatchResourcesClient { }), }); + // If it's a new connector, show the "Source imported successfully" modal + const isConnectorModel = + res.resource.meta?.filePaths?.[0]?.startsWith("/connectors/"); + const isSourceConnector = + res.resource.meta?.filePaths?.[0]?.startsWith("/sources/"); + const isNewConnector = + (isConnectorModel || isSourceConnector) && + res.resource.meta.specVersion === "1" && // First file version + res.resource.meta.stateVersion === "2" && // First ingest is complete + (await isLeafResource(res.resource, this.instanceId)); // Protects against existing projects reconciling anew + if (isNewConnector) { + const filePath = res.resource?.meta?.filePaths?.[0] as string; + sourceImportedPath.set(filePath); + } + // Done return; case ResourceKind.Source: case ResourceKind.Model: { + console.log("model: ", res.resource); // TODO: differentiate between a Model's executorConnector and resultConnector const connectorName = (res.name.kind as ResourceKind) === ResourceKind.Source From f225bcd26226eb4befc03b576fe7e2eeaaa2adb2 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 6 Aug 2025 16:56:21 -0700 Subject: [PATCH 08/21] lint --- .../src/features/entity-management/WatchResourcesClient.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web-common/src/features/entity-management/WatchResourcesClient.ts b/web-common/src/features/entity-management/WatchResourcesClient.ts index acafade957c..0da6e07b8de 100644 --- a/web-common/src/features/entity-management/WatchResourcesClient.ts +++ b/web-common/src/features/entity-management/WatchResourcesClient.ts @@ -119,7 +119,7 @@ export class WatchResourcesClient { }); switch (res.name.kind as ResourceKind) { - case ResourceKind.Connector: + case ResourceKind.Connector: { console.log("connector: ", res.resource); // Invalidate the list of connectors void queryClient.invalidateQueries({ @@ -153,6 +153,7 @@ export class WatchResourcesClient { // Done return; + } case ResourceKind.Source: case ResourceKind.Model: { From e73f867bdf07eb68536de4f193938bbb19224201 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 6 Aug 2025 18:45:15 -0700 Subject: [PATCH 09/21] fix top of file local file yaml compilation --- web-common/src/features/sources/sourceUtils.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/web-common/src/features/sources/sourceUtils.ts b/web-common/src/features/sources/sourceUtils.ts index d2dd7d56ee5..82d0f55d1f6 100644 --- a/web-common/src/features/sources/sourceUtils.ts +++ b/web-common/src/features/sources/sourceUtils.ts @@ -9,11 +9,17 @@ import { makeDotEnvConnectorKey } from "../connectors/code-utils"; import { sanitizeEntityName } from "../entity-management/name-utils"; // Helper text that we put at the top of every Source YAML file -const TOP_OF_FILE = `# Connector YAML +const CONNECTOR_FILE_TOP = `# Connector YAML # Reference documentation: https://docs.rilldata.com/reference/project-files/connectors type: connector`; +const LOCAL_FILE_TOP = `# Source YAML +# Reference documentation: https://docs.rilldata.com/reference/project-files/sources + +type: model +materialize: true`; + export function compileSourceYAML( connector: V1ConnectorDriver, formValues: Record, @@ -72,12 +78,13 @@ export function compileSourceYAML( }) .join("\n"); - // Return the compiled YAML - return `${TOP_OF_FILE}\n\ndriver: ${connector.name}\n` + compiledKeyValues; + return ( + `${CONNECTOR_FILE_TOP}\n\ndriver: ${connector.name}\n` + compiledKeyValues + ); } export function compileLocalFileSourceYAML(path: string) { - return `${TOP_OF_FILE}\n\ndriver: duckdb\nsql: "${buildDuckDbQuery(path)}"`; + return `${LOCAL_FILE_TOP}\n\ndriver: duckdb\nsql: "${buildDuckDbQuery(path)}"`; } function buildDuckDbQuery(path: string): string { From 4e4e59751eabedfaee2ab57b753b5ff4d801ac08 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 6 Aug 2025 18:49:03 -0700 Subject: [PATCH 10/21] rm connector set imported file path --- .../entity-management/WatchResourcesClient.ts | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/web-common/src/features/entity-management/WatchResourcesClient.ts b/web-common/src/features/entity-management/WatchResourcesClient.ts index 0da6e07b8de..a62b2492a54 100644 --- a/web-common/src/features/entity-management/WatchResourcesClient.ts +++ b/web-common/src/features/entity-management/WatchResourcesClient.ts @@ -136,21 +136,6 @@ export class WatchResourcesClient { }), }); - // If it's a new connector, show the "Source imported successfully" modal - const isConnectorModel = - res.resource.meta?.filePaths?.[0]?.startsWith("/connectors/"); - const isSourceConnector = - res.resource.meta?.filePaths?.[0]?.startsWith("/sources/"); - const isNewConnector = - (isConnectorModel || isSourceConnector) && - res.resource.meta.specVersion === "1" && // First file version - res.resource.meta.stateVersion === "2" && // First ingest is complete - (await isLeafResource(res.resource, this.instanceId)); // Protects against existing projects reconciling anew - if (isNewConnector) { - const filePath = res.resource?.meta?.filePaths?.[0] as string; - sourceImportedPath.set(filePath); - } - // Done return; } From 573f4b8665acb7068b2cfe6728c73fdd17689923 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 7 Aug 2025 14:31:25 -0700 Subject: [PATCH 11/21] revert --- runtime/drivers/gcs/gcs.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/runtime/drivers/gcs/gcs.go b/runtime/drivers/gcs/gcs.go index 1b42a6a4e97..9da60510ff5 100644 --- a/runtime/drivers/gcs/gcs.go +++ b/runtime/drivers/gcs/gcs.go @@ -44,6 +44,14 @@ var spec = drivers.Spec{ Required: true, Hint: "Glob patterns are supported", }, + { + Key: "name", + Type: drivers.StringPropertyType, + DisplayName: "Source name", + Description: "The name of the source", + Placeholder: "my_new_source", + Required: true, + }, }, ImplementsObjectStore: true, } From 47d24d3e96a401a9669e73dab57272fded735449 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 7 Aug 2025 15:35:13 -0700 Subject: [PATCH 12/21] clean up --- .../src/features/entity-management/WatchResourcesClient.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/web-common/src/features/entity-management/WatchResourcesClient.ts b/web-common/src/features/entity-management/WatchResourcesClient.ts index a62b2492a54..796528cbc6e 100644 --- a/web-common/src/features/entity-management/WatchResourcesClient.ts +++ b/web-common/src/features/entity-management/WatchResourcesClient.ts @@ -119,8 +119,7 @@ export class WatchResourcesClient { }); switch (res.name.kind as ResourceKind) { - case ResourceKind.Connector: { - console.log("connector: ", res.resource); + case ResourceKind.Connector: // Invalidate the list of connectors void queryClient.invalidateQueries({ queryKey: getRuntimeServiceAnalyzeConnectorsQueryKey( @@ -138,11 +137,9 @@ export class WatchResourcesClient { // Done return; - } case ResourceKind.Source: case ResourceKind.Model: { - console.log("model: ", res.resource); // TODO: differentiate between a Model's executorConnector and resultConnector const connectorName = (res.name.kind as ResourceKind) === ResourceKind.Source From ec6679b7a60f9db22343b12e976f49e5d386ca8e Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 7 Aug 2025 16:31:09 -0700 Subject: [PATCH 13/21] tweak connector form type --- .../src/features/sources/modal/AddDataModal.svelte | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataModal.svelte b/web-common/src/features/sources/modal/AddDataModal.svelte index cbd7d3381ae..fb5d9b1ba7b 100644 --- a/web-common/src/features/sources/modal/AddDataModal.svelte +++ b/web-common/src/features/sources/modal/AddDataModal.svelte @@ -173,6 +173,11 @@ $: isModelingSupportedForDefaultOlapDriver = useIsModelingSupportedForDefaultOlapDriver($runtime.instanceId); $: isModelingSupported = $isModelingSupportedForDefaultOlapDriver.data; + + $: isConnectorType = + selectedConnector?.implementsOlap || + selectedConnector?.implementsSqlStore || + selectedConnector?.implementsWarehouse; {#if step >= 1 || $duplicateSourceName} @@ -276,9 +281,7 @@ {:else if selectedConnector.name} Date: Fri, 8 Aug 2025 13:40:53 -0700 Subject: [PATCH 14/21] modify bigquery configproperties, yupschemas --- runtime/drivers/bigquery/bigquery.go | 43 +++---------------- .../src/features/sources/modal/yupSchemas.ts | 8 +--- 2 files changed, 8 insertions(+), 43 deletions(-) diff --git a/runtime/drivers/bigquery/bigquery.go b/runtime/drivers/bigquery/bigquery.go index a0a8db17349..0dee5df4771 100644 --- a/runtime/drivers/bigquery/bigquery.go +++ b/runtime/drivers/bigquery/bigquery.go @@ -28,53 +28,22 @@ var spec = drivers.Spec{ DocsURL: "https://docs.rilldata.com/reference/connectors/bigquery", ConfigProperties: []*drivers.PropertySpec{ { - Key: "google_application_credentials", - Type: drivers.FilePropertyType, - Hint: "Enter path of file to load from.", + Key: "google_application_credentials", + Type: drivers.InformationalPropertyType, + DisplayName: "GCP credentials", + Description: "GCP credentials inferred from your local environment.", + Hint: "Set your local credentials: gcloud auth application-default login Click to learn more.", + DocsURL: "https://docs.rilldata.com/reference/connectors/gcs#local-credentials", }, { Key: "project_id", Type: drivers.StringPropertyType, Required: false, DisplayName: "Project ID", - Description: "Default Google project ID.", - }, - }, - // Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas. - SourceProperties: []*drivers.PropertySpec{ - { - Key: "sql", - Type: drivers.StringPropertyType, - Required: true, - DisplayName: "SQL", - Description: "Query to extract data from BigQuery.", - Placeholder: "select * from project.dataset.table;", - }, - { - Key: "project_id", - Type: drivers.StringPropertyType, - Required: true, - DisplayName: "Project ID", Description: "Google project ID.", Placeholder: "my-project", Hint: "Rill will use the project ID from your local credentials, unless set here. Set this if no project ID configured in credentials.", }, - { - Key: "name", - Type: drivers.StringPropertyType, - DisplayName: "Source name", - Description: "The name of the source", - Placeholder: "my_new_source", - Required: true, - }, - { - Key: "google_application_credentials", - Type: drivers.InformationalPropertyType, - DisplayName: "GCP credentials", - Description: "GCP credentials inferred from your local environment.", - Hint: "Set your local credentials: gcloud auth application-default login Click to learn more.", - DocsURL: "https://docs.rilldata.com/reference/connectors/gcs#local-credentials", - }, }, ImplementsWarehouse: true, } diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index ca6f1901dec..aa55b302dbe 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -64,12 +64,8 @@ export const getYupSchema = { }), bigquery: yup.object().shape({ - sql: yup.string().required("sql is required"), - project_id: yup.string().required("project_id is required"), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), + google_application_credentials: yup.string().optional(), + project_id: yup.string().optional(), }), azure: yup.object().shape({ From 1c1e3f7cbde98f7f2c3cf0d87917587acfe652b1 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 8 Aug 2025 14:01:01 -0700 Subject: [PATCH 15/21] only test connection for the previous olap connectors --- .../sources/modal/submitAddDataForm.ts | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/web-common/src/features/sources/modal/submitAddDataForm.ts b/web-common/src/features/sources/modal/submitAddDataForm.ts index 5f7f81acb56..0f358d20b9d 100644 --- a/web-common/src/features/sources/modal/submitAddDataForm.ts +++ b/web-common/src/features/sources/modal/submitAddDataForm.ts @@ -172,19 +172,25 @@ export async function submitAddOLAPConnectorForm( throw new Error(errorMessage); } - // Test the connection to the OLAP database + // Test the connection to the OLAP database (only for original OLAP connectors) // If the connection test fails, rollback the changes - const result = await testOLAPConnector(instanceId, connector.name as string); - if (!result.success) { - await rollbackConnectorChanges( + const OLAP_CONNECTORS = ["clickhouse", "druid", "pinot"]; + if (OLAP_CONNECTORS.includes(connector.name as string)) { + const result = await testOLAPConnector( instanceId, - newConnectorFilePath, - originalEnvBlob, + connector.name as string, ); - throw { - message: result.error || "Unable to establish a connection", - details: result.details, - }; + if (!result.success) { + await rollbackConnectorChanges( + instanceId, + newConnectorFilePath, + originalEnvBlob, + ); + throw { + message: result.error || "Unable to establish a connection", + details: result.details, + }; + } } /** From e67e4636bfb23cc17a9c76a88e501219d7e8ae32 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 8 Aug 2025 14:02:27 -0700 Subject: [PATCH 16/21] centralize sources and olap connectors to constants --- .../sources/modal/AddDataModal.svelte | 23 +------------------ .../src/features/sources/modal/constants.ts | 22 ++++++++++++++++++ .../sources/modal/submitAddDataForm.ts | 2 +- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataModal.svelte b/web-common/src/features/sources/modal/AddDataModal.svelte index fb5d9b1ba7b..22e063e39af 100644 --- a/web-common/src/features/sources/modal/AddDataModal.svelte +++ b/web-common/src/features/sources/modal/AddDataModal.svelte @@ -39,34 +39,13 @@ import DuplicateSource from "./DuplicateSource.svelte"; import LocalSourceUpload from "./LocalSourceUpload.svelte"; import RequestConnectorForm from "./RequestConnectorForm.svelte"; + import { OLAP_CONNECTORS, SORT_ORDER, SOURCES } from "./constants"; let step = 0; let selectedConnector: null | V1ConnectorDriver = null; let requestConnector = false; let isSubmittingForm = false; - const SOURCES = [ - "gcs", - "s3", - "azure", - "bigquery", - "athena", - "redshift", - "duckdb", - "motherduck", - "postgres", - "mysql", - "sqlite", - "snowflake", - "salesforce", - "local_file", - "https", - ]; - - const OLAP_CONNECTORS = ["clickhouse", "druid", "pinot"]; - - const SORT_ORDER = [...SOURCES, ...OLAP_CONNECTORS]; - const ICONS = { gcs: GoogleCloudStorage, s3: AmazonS3, diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 6e7ff26d14e..eeacce3c0e1 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -10,3 +10,25 @@ export const CONNECTION_TAB_OPTIONS: { value: string; label: string }[] = [ { value: "parameters", label: "Enter parameters" }, { value: "dsn", label: "Enter connection string" }, ]; + +export const SOURCES = [ + "gcs", + "s3", + "azure", + "bigquery", + "athena", + "redshift", + "duckdb", + "motherduck", + "postgres", + "mysql", + "sqlite", + "snowflake", + "salesforce", + "local_file", + "https", +]; + +export const OLAP_CONNECTORS = ["clickhouse", "druid", "pinot"]; + +export const SORT_ORDER = [...SOURCES, ...OLAP_CONNECTORS]; diff --git a/web-common/src/features/sources/modal/submitAddDataForm.ts b/web-common/src/features/sources/modal/submitAddDataForm.ts index 0f358d20b9d..629513483e4 100644 --- a/web-common/src/features/sources/modal/submitAddDataForm.ts +++ b/web-common/src/features/sources/modal/submitAddDataForm.ts @@ -32,6 +32,7 @@ import { EntityType } from "../../entity-management/types"; import { EMPTY_PROJECT_TITLE } from "../../welcome/constants"; import { isProjectInitialized } from "../../welcome/is-project-initialized"; import { compileSourceYAML, maybeRewriteToDuckDb } from "../sourceUtils"; +import { OLAP_CONNECTORS } from "./constants"; interface AddDataFormValues { // name: string; // Commenting out until we add user-provided names for Connectors @@ -174,7 +175,6 @@ export async function submitAddOLAPConnectorForm( // Test the connection to the OLAP database (only for original OLAP connectors) // If the connection test fails, rollback the changes - const OLAP_CONNECTORS = ["clickhouse", "druid", "pinot"]; if (OLAP_CONNECTORS.includes(connector.name as string)) { const result = await testOLAPConnector( instanceId, From d5fc2efa4ab587c01cbfe4c660c9be524566a328 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 8 Aug 2025 14:10:13 -0700 Subject: [PATCH 17/21] revert for sanity --- web-common/src/features/sources/modal/AddDataModal.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web-common/src/features/sources/modal/AddDataModal.svelte b/web-common/src/features/sources/modal/AddDataModal.svelte index 22e063e39af..365eb9ebcd2 100644 --- a/web-common/src/features/sources/modal/AddDataModal.svelte +++ b/web-common/src/features/sources/modal/AddDataModal.svelte @@ -260,7 +260,9 @@ {:else if selectedConnector.name} Date: Fri, 8 Aug 2025 14:40:54 -0700 Subject: [PATCH 18/21] only set olap connector in rill for olap engines --- .../sources/modal/AddDataModal.svelte | 4 +--- .../src/features/sources/modal/constants.ts | 2 ++ .../sources/modal/submitAddDataForm.ts | 20 ++++++++++++------- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataModal.svelte b/web-common/src/features/sources/modal/AddDataModal.svelte index 365eb9ebcd2..22e063e39af 100644 --- a/web-common/src/features/sources/modal/AddDataModal.svelte +++ b/web-common/src/features/sources/modal/AddDataModal.svelte @@ -260,9 +260,7 @@ {:else if selectedConnector.name} Date: Fri, 8 Aug 2025 14:42:58 -0700 Subject: [PATCH 19/21] clean up --- web-common/src/features/sources/modal/AddDataModal.svelte | 8 ++++---- web-common/src/features/sources/modal/constants.ts | 8 ++++---- .../src/features/sources/modal/submitAddDataForm.ts | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataModal.svelte b/web-common/src/features/sources/modal/AddDataModal.svelte index 22e063e39af..11811fa95af 100644 --- a/web-common/src/features/sources/modal/AddDataModal.svelte +++ b/web-common/src/features/sources/modal/AddDataModal.svelte @@ -39,7 +39,7 @@ import DuplicateSource from "./DuplicateSource.svelte"; import LocalSourceUpload from "./LocalSourceUpload.svelte"; import RequestConnectorForm from "./RequestConnectorForm.svelte"; - import { OLAP_CONNECTORS, SORT_ORDER, SOURCES } from "./constants"; + import { OLAP_ENGINES, SORT_ORDER, SOURCES } from "./constants"; let step = 0; let selectedConnector: null | V1ConnectorDriver = null; @@ -75,10 +75,10 @@ data.connectors && data.connectors .filter( - // Only show connectors in SOURCES or OLAP_CONNECTORS + // Only show connectors in SOURCES or OLAP_ENGINES (a) => a.name && - (SOURCES.includes(a.name) || OLAP_CONNECTORS.includes(a.name)), + (SOURCES.includes(a.name) || OLAP_ENGINES.includes(a.name)), ) .sort( // CAST SAFETY: we have filtered out any connectors that @@ -205,7 +205,7 @@ Connect an OLAP engine
- {#each connectors?.filter((c) => c.name && OLAP_CONNECTORS.includes(c.name)) as connector (connector.name)} + {#each connectors?.filter((c) => c.name && OLAP_ENGINES.includes(c.name)) as connector (connector.name)} {#if connector.name}