Skip to content

Create connector files #7726

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 27 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
16b63cf
create postgres connector init
lovincyrus Aug 5, 2025
8859c2a
remove sql and name from postgres driver
lovincyrus Aug 5, 2025
2a25bf6
remove sql and name from mysql
lovincyrus Aug 5, 2025
0190cb0
remove name from gcs driver
lovincyrus Aug 5, 2025
e3e9457
remove s3 from rewrite to duckdb
lovincyrus Aug 5, 2025
d0540ea
Revert "remove s3 from rewrite to duckdb"
lovincyrus Aug 6, 2025
570b26a
set source imported path for connector case
lovincyrus Aug 6, 2025
c07e489
lint
lovincyrus Aug 6, 2025
7b50df4
fix top of file local file yaml compilation
lovincyrus Aug 7, 2025
dfbbb5c
rm connector set imported file path
lovincyrus Aug 7, 2025
23e47f6
revert
lovincyrus Aug 7, 2025
f894f52
clean up
lovincyrus Aug 7, 2025
056590c
tweak connector form type
lovincyrus Aug 7, 2025
bccd757
modify bigquery configproperties, yupschemas
lovincyrus Aug 8, 2025
1ab93f4
only test connection for the previous olap connectors
lovincyrus Aug 8, 2025
fdfe069
centralize sources and olap connectors to constants
lovincyrus Aug 8, 2025
8db3ce5
revert for sanity
lovincyrus Aug 8, 2025
d1c9836
only set olap connector in rill for olap engines
lovincyrus Aug 8, 2025
5d80b68
clean up
lovincyrus Aug 8, 2025
41faaf6
fix config properties for athena
lovincyrus Aug 8, 2025
d43ddb2
fix config properties for redshift
lovincyrus Aug 8, 2025
f7ac460
update config properties for snowflake
lovincyrus Aug 11, 2025
f10ca90
tweak form to handle connector with dsn only
lovincyrus Aug 11, 2025
9696fb2
fix salesforce yupSchema
lovincyrus Aug 11, 2025
be23a6f
remove source properties of mysql and postgres
lovincyrus Aug 11, 2025
93fa241
remove source properties from redshift
lovincyrus Aug 11, 2025
3a258a9
fix source properties of duckdb
lovincyrus Aug 11, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 10 additions & 39 deletions runtime/drivers/athena/athena.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,56 +34,27 @@ var spec = drivers.Spec{
DocsURL: "https://docs.rilldata.com/connect/data-source/athena",
ConfigProperties: []*drivers.PropertySpec{
{
Key: "aws_access_key_id",
Type: drivers.StringPropertyType,
Secret: true,
Key: "aws_access_key_id",
Type: drivers.StringPropertyType,
DisplayName: "AWS access key ID",
Description: "AWS access key ID",
Required: true,
Secret: true,
},
{
Key: "aws_secret_access_key",
Type: drivers.StringPropertyType,
Secret: true,
},
},
// Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas.
SourceProperties: []*drivers.PropertySpec{
{
Key: "sql",
Key: "aws_secret_access_key",
Type: drivers.StringPropertyType,
DisplayName: "AWS secret access key",
Description: "AWS secret access key",
Required: true,
DisplayName: "SQL",
Description: "Query to extract data from Athena.",
Placeholder: "select * from catalog.table;",
Secret: true,
},
{
Key: "output_location",
Type: drivers.StringPropertyType,
DisplayName: "S3 output location",
Description: "An output location for query result is required either through the workgroup result configuration setting or set here.",
Placeholder: "s3://bucket-name/path/",
Required: false,
},
{
Key: "workgroup",
Type: drivers.StringPropertyType,
DisplayName: "AWS Athena workgroup",
Description: "AWS Athena workgroup to use for queries. Default is primary",
Placeholder: "primary",
Required: false,
},
{
Key: "region",
Type: drivers.StringPropertyType,
DisplayName: "AWS region",
Description: "AWS region to connect to Athena and the output location.",
Placeholder: "us-east-1",
Required: false,
},
{
Key: "name",
Type: drivers.StringPropertyType,
DisplayName: "Source name",
Description: "The name of the source",
Placeholder: "my_new_source",
Required: true,
},
},
Expand Down
43 changes: 6 additions & 37 deletions runtime/drivers/bigquery/bigquery.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,53 +28,22 @@ var spec = drivers.Spec{
DocsURL: "https://docs.rilldata.com/connect/data-source/bigquery",
ConfigProperties: []*drivers.PropertySpec{
{
Key: "google_application_credentials",
Type: drivers.FilePropertyType,
Hint: "Enter path of file to load from.",
Key: "google_application_credentials",
Type: drivers.InformationalPropertyType,
DisplayName: "GCP credentials",
Description: "GCP credentials inferred from your local environment.",
Hint: "Set your local credentials: <code>gcloud auth application-default login</code> Click to learn more.",
DocsURL: "https://docs.rilldata.com/connect/data-source/gcs#rill-developer-local-credentials",
},
{
Key: "project_id",
Type: drivers.StringPropertyType,
Required: false,
DisplayName: "Project ID",
Description: "Default Google project ID.",
},
},
// Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas.
SourceProperties: []*drivers.PropertySpec{
{
Key: "sql",
Type: drivers.StringPropertyType,
Required: true,
DisplayName: "SQL",
Description: "Query to extract data from BigQuery.",
Placeholder: "select * from project.dataset.table;",
},
{
Key: "project_id",
Type: drivers.StringPropertyType,
Required: true,
DisplayName: "Project ID",
Description: "Google project ID.",
Placeholder: "my-project",
Hint: "Rill will use the project ID from your local credentials, unless set here. Set this if no project ID configured in credentials.",
},
{
Key: "name",
Type: drivers.StringPropertyType,
DisplayName: "Source name",
Description: "The name of the source",
Placeholder: "my_new_source",
Required: true,
},
{
Key: "google_application_credentials",
Type: drivers.InformationalPropertyType,
DisplayName: "GCP credentials",
Description: "GCP credentials inferred from your local environment.",
Hint: "Set your local credentials: <code>gcloud auth application-default login</code> Click to learn more.",
DocsURL: "https://docs.rilldata.com/connect/data-source/gcs#rill-developer-local-credentials",
},
},
ImplementsWarehouse: true,
}
Expand Down
27 changes: 0 additions & 27 deletions runtime/drivers/duckdb/duckdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,33 +49,6 @@ var spec = drivers.Spec{
Placeholder: "/path/to/main.db",
},
},
// Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas.
SourceProperties: []*drivers.PropertySpec{
{
Key: "db",
Type: drivers.StringPropertyType,
Required: true,
DisplayName: "DB",
Description: "Path to DuckDB database",
Placeholder: "/path/to/duckdb.db",
},
{
Key: "sql",
Type: drivers.StringPropertyType,
Required: true,
DisplayName: "SQL",
Description: "Query to extract data from DuckDB.",
Placeholder: "select * from table;",
},
{
Key: "name",
Type: drivers.StringPropertyType,
DisplayName: "Source name",
Description: "The name of the source",
Placeholder: "my_new_source",
Required: true,
},
},
ImplementsOLAP: true,
}

Expand Down
27 changes: 1 addition & 26 deletions runtime/drivers/mysql/mysql.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,41 +27,16 @@ var spec = drivers.Spec{
Description: "Connect to MySQL.",
DocsURL: "https://docs.rilldata.com/connect/data-source/mysql",
ConfigProperties: []*drivers.PropertySpec{
{
Key: "dsn",
Type: drivers.StringPropertyType,
Placeholder: "mysql://user:password@host:3306/my-db",
Secret: true,
},
},
// Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas.
SourceProperties: []*drivers.PropertySpec{
{
Key: "sql",
Type: drivers.StringPropertyType,
Required: true,
DisplayName: "SQL",
Description: "Query to extract data from MySQL.",
Placeholder: "select * from table;",
},
{
Key: "dsn",
Type: drivers.StringPropertyType,
DisplayName: "MySQL Connection String",
Required: false,
Required: true,
DocsURL: "https://dev.mysql.com/doc/refman/8.4/en/connecting-using-uri-or-key-value-pairs.html#connecting-using-uri",
Placeholder: "mysql://user:password@host:3306/my-db",
Hint: "Can be configured here or by setting the 'connector.mysql.dsn' environment variable (using '.env' or '--env')",
Secret: true,
},
{
Key: "name",
Type: drivers.StringPropertyType,
DisplayName: "Source name",
Description: "The name of the source",
Placeholder: "my_new_source",
Required: true,
},
},
ImplementsSQLStore: true,
}
Expand Down
25 changes: 1 addition & 24 deletions runtime/drivers/postgres/postgres.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,39 +27,16 @@ var spec = drivers.Spec{
Description: "Connect to Postgres.",
DocsURL: "https://docs.rilldata.com/connect/data-source/postgres",
ConfigProperties: []*drivers.PropertySpec{
{
Key: "database_url",
Secret: true,
},
},
// Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas.
SourceProperties: []*drivers.PropertySpec{
{
Key: "sql",
Type: drivers.StringPropertyType,
Required: true,
DisplayName: "SQL",
Description: "Query to extract data from Postgres.",
Placeholder: "select * from table;",
},
{
Key: "database_url",
Type: drivers.StringPropertyType,
DisplayName: "Postgres Connection String",
Required: false,
Required: true,
DocsURL: "https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING",
Placeholder: "postgresql://postgres:postgres@localhost:5432/postgres",
Hint: "Can be configured here or by setting the 'connector.postgres.database_url' environment variable (using '.env' or '--env')",
Secret: true,
},
{
Key: "name",
Type: drivers.StringPropertyType,
DisplayName: "Source name",
Description: "The name of the source",
Placeholder: "my_new_source",
Required: true,
},
},
ImplementsSQLStore: true,
}
Expand Down
53 changes: 8 additions & 45 deletions runtime/drivers/redshift/redshift.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,33 +31,20 @@ var spec = drivers.Spec{
DocsURL: "https://docs.rilldata.com/connect/data-source/redshift",
ConfigProperties: []*drivers.PropertySpec{
{
Key: "aws_access_key_id",
Type: drivers.StringPropertyType,
Secret: true,
},
{
Key: "aws_secret_access_key",
Type: drivers.StringPropertyType,
Secret: true,
},
},
// Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas.
SourceProperties: []*drivers.PropertySpec{
{
Key: "sql",
Key: "aws_access_key_id",
Type: drivers.StringPropertyType,
DisplayName: "AWS access key ID",
Description: "AWS access key ID",
Required: true,
DisplayName: "SQL",
Description: "Query to extract data from Redshift.",
Placeholder: "select * from public.table;",
Secret: true,
},
{
Key: "output_location",
Key: "aws_secret_access_key",
Type: drivers.StringPropertyType,
DisplayName: "S3 output location",
Description: "Output location in S3 for temporary data.",
Placeholder: "s3://bucket-name/path/",
DisplayName: "AWS secret access key",
Description: "AWS secret access key",
Required: true,
Secret: true,
},
{
Key: "workgroup",
Expand All @@ -83,30 +70,6 @@ var spec = drivers.Spec{
Placeholder: "dev",
Required: true,
},
{
Key: "cluster_identifier",
Type: drivers.StringPropertyType,
DisplayName: "Redshift cluster identifier",
Description: "Redshift cluster identifier",
Placeholder: "redshift-cluster-1",
Required: false,
},
{
Key: "role_arn",
Type: drivers.StringPropertyType,
DisplayName: "Redshift role ARN",
Description: "Redshift role ARN",
Placeholder: "arn:aws:iam::03214372:role/service-role/AmazonRedshift-CommandsAccessRole-20240307T203902",
Required: true,
},
{
Key: "name",
Type: drivers.StringPropertyType,
DisplayName: "Source name",
Description: "The name of the source",
Placeholder: "my_new_source",
Required: true,
},
},
ImplementsWarehouse: true,
}
Expand Down
Loading