From 0d8d73d870653a2a81c42c9dd5916beaf5f794dd Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Wed, 20 Sep 2023 17:22:36 -0700 Subject: [PATCH 1/8] feat(vet): Support managed databases --- internal/cmd/vet.go | 66 ++++++++++++++++++++++++++++++++++++- internal/config/config.go | 3 +- internal/config/validate.go | 4 +-- 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/internal/cmd/vet.go b/internal/cmd/vet.go index da0189e9d9..ed7b5937b7 100644 --- a/internal/cmd/vet.go +++ b/internal/cmd/vet.go @@ -26,6 +26,8 @@ import ( "github.com/sqlc-dev/sqlc/internal/debug" "github.com/sqlc-dev/sqlc/internal/opts" "github.com/sqlc-dev/sqlc/internal/plugin" + "github.com/sqlc-dev/sqlc/internal/quickdb" + pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1" "github.com/sqlc-dev/sqlc/internal/shfmt" "github.com/sqlc-dev/sqlc/internal/vet" ) @@ -376,6 +378,62 @@ type checker struct { Envmap map[string]string Stderr io.Writer NoDatabase bool + Client pb.QuickClient +} + +func (c *checker) fetchDatabaseUri(ctx context.Context, s config.SQL) (string, func() error, error) { + cleanup := func() error { + return nil + } + + if s.Database == nil { + panic("fetch database URI called with nil database") + } + if !s.Database.Managed { + uri, err := c.DSN(s.Database.URI) + return uri, cleanup, err + } + if s.Engine != config.EnginePostgreSQL { + return "", cleanup, fmt.Errorf("managed: only PostgreSQL currently") + } + + if c.Client == nil { + // FIXME: Eventual race condition + client, err := quickdb.NewClientFromConfig(c.Conf.Cloud) + if err != nil { + return "", cleanup, fmt.Errorf("managed: client: %w", err) + } + c.Client = client + } + + var migrations []string + for _, query := range s.Schema { + contents, err := os.ReadFile(query) + if err != nil { + return "", cleanup, fmt.Errorf("read file: %w", err) + } + migrations = append(migrations, string(contents)) + } + + start := time.Now() + resp, err := c.Client.CreateEphemeralDatabase(ctx, &pb.CreateEphemeralDatabaseRequest{ + Engine: "postgresql", + Region: "sjc", + Migrations: migrations, + }) + if err != nil { + return "", cleanup, fmt.Errorf("managed: create database: %w", err) + } + + cleanup = func() error { + _, err := c.Client.DropEphemeralDatabase(ctx, &pb.DropEphemeralDatabaseRequest{ + DatabaseId: resp.DatabaseId, + }) + return err + } + + fmt.Println("createdb+template", time.Since(start)) + return resp.Uri, cleanup, nil } func (c *checker) DSN(dsn string) (string, error) { @@ -422,10 +480,16 @@ func (c *checker) checkSQL(ctx context.Context, s config.SQL) error { if c.NoDatabase { return fmt.Errorf("database: connections disabled via command line flag") } - dburl, err := c.DSN(s.Database.URI) + dburl, cleanup, err := c.fetchDatabaseUri(ctx, s) if err != nil { return err } + defer func() { + if err := cleanup(); err != nil { + fmt.Fprintf(c.Stderr, "error cleaning up: %s\n", err) + } + }() + switch s.Engine { case config.EnginePostgreSQL: conn, err := pgx.Connect(ctx, dburl) diff --git a/internal/config/config.go b/internal/config/config.go index 35270fbba0..d28ec0e62d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -69,7 +69,8 @@ type Project struct { } type Database struct { - URI string `json:"uri" yaml:"uri"` + URI string `json:"uri" yaml:"uri"` + Managed bool `json:"managed" yaml:"managed"` } type Cloud struct { diff --git a/internal/config/validate.go b/internal/config/validate.go index e0e056fd65..207a888ecf 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -12,8 +12,8 @@ func Validate(c *Config) error { return fmt.Errorf("invalid config: emit_methods_with_db_argument and emit_prepared_queries settings are mutually exclusive") } if sql.Database != nil { - if sql.Database.URI == "" { - return fmt.Errorf("invalid config: database must have a non-empty URI") + if sql.Database.URI == "" && !sql.Database.Managed { + return fmt.Errorf("invalid config: database must be managed or have a non-empty URI") } } } From 8dfde57dafa749c47961614aa6196730ecc51625 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Wed, 20 Sep 2023 17:45:57 -0700 Subject: [PATCH 2/8] feat(vet): Run rules against a managed database --- .github/workflows/ci.yml | 16 +--------------- examples/authors/sqlc.yaml | 4 +++- examples/batch/sqlc.json | 5 ++++- examples/booktest/sqlc.json | 5 ++++- examples/jets/sqlc.json | 5 ++++- examples/ondeck/sqlc.json | 5 ++++- internal/cmd/cmd.go | 4 ++++ internal/cmd/generate.go | 2 +- internal/cmd/vet.go | 4 +--- internal/endtoend/vet_test.go | 6 +----- 10 files changed, 27 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26bf764df8..845cb5c2a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,16 +32,6 @@ jobs: runs-on: ubuntu-latest services: - postgres: - image: "postgres:15" - env: - POSTGRES_DB: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_USER: postgres - ports: - - 5432:5432 - # needed because the postgres container does not provide a healthcheck - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 mysql: image: "mysql/mysql-server:8.0" env: @@ -69,17 +59,13 @@ jobs: - name: test ./... run: gotestsum --junitfile junit.xml -- --tags=examples ./... env: - PG_USER: postgres - PG_HOST: localhost - PG_DATABASE: postgres - PG_PASSWORD: postgres - PG_PORT: ${{ job.services.postgres.ports['5432'] }} MYSQL_DATABASE: mysql MYSQL_HOST: localhost MYSQL_PORT: ${{ job.services.mysql.ports['3306'] }} MYSQL_ROOT_PASSWORD: mysecretpassword CI_SQLC_PROJECT_ID: ${{ secrets.CI_SQLC_PROJECT_ID }} CI_SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }} + SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }} - name: build internal/endtoend run: go build ./... diff --git a/examples/authors/sqlc.yaml b/examples/authors/sqlc.yaml index d43fb976d6..8270782edd 100644 --- a/examples/authors/sqlc.yaml +++ b/examples/authors/sqlc.yaml @@ -1,10 +1,12 @@ version: '2' +cloud: + project: "01HAQMMECEYQYKFJN8MP16QC41" sql: - schema: postgresql/schema.sql queries: postgresql/query.sql engine: postgresql database: - uri: postgresql://${PG_USER}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/authors + managed: true rules: - sqlc/db-prepare - postgresql-query-too-costly diff --git a/examples/batch/sqlc.json b/examples/batch/sqlc.json index dfd7ac8099..8c27bc540c 100644 --- a/examples/batch/sqlc.json +++ b/examples/batch/sqlc.json @@ -1,5 +1,8 @@ { "version": "1", + "cloud": { + "project": "01HAQMMECEYQYKFJN8MP16QC41" + }, "packages": [ { "path": "postgresql", @@ -8,7 +11,7 @@ "queries": "postgresql/query.sql", "engine": "postgresql", "database": { - "uri": "postgresql://${PG_USER}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/batch" + "managed": true }, "rules": [ "sqlc/db-prepare" diff --git a/examples/booktest/sqlc.json b/examples/booktest/sqlc.json index 8220d36010..72cd08936d 100644 --- a/examples/booktest/sqlc.json +++ b/examples/booktest/sqlc.json @@ -1,5 +1,8 @@ { "version": "1", + "cloud": { + "project": "01HAQMMECEYQYKFJN8MP16QC41" + }, "packages": [ { "name": "booktest", @@ -8,7 +11,7 @@ "queries": "postgresql/query.sql", "engine": "postgresql", "database": { - "uri": "postgresql://${PG_USER}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/booktest" + "managed": true }, "rules": [ "sqlc/db-prepare" diff --git a/examples/jets/sqlc.json b/examples/jets/sqlc.json index f8c5ef1ba7..85a35a4f7a 100644 --- a/examples/jets/sqlc.json +++ b/examples/jets/sqlc.json @@ -1,5 +1,8 @@ { "version": "1", + "cloud": { + "project": "01HAQMMECEYQYKFJN8MP16QC41" + }, "packages": [ { "path": "postgresql", @@ -8,7 +11,7 @@ "queries": "postgresql/query-building.sql", "engine": "postgresql", "database": { - "uri": "postgresql://${PG_USER}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/jets" + "managed": true }, "rules": [ "sqlc/db-prepare" diff --git a/examples/ondeck/sqlc.json b/examples/ondeck/sqlc.json index 61862a2a06..c9290db568 100644 --- a/examples/ondeck/sqlc.json +++ b/examples/ondeck/sqlc.json @@ -1,5 +1,8 @@ { "version": "1", + "cloud": { + "project": "01HAQMMECEYQYKFJN8MP16QC41" + }, "packages": [ { "path": "postgresql", @@ -8,7 +11,7 @@ "queries": "postgresql/query", "engine": "postgresql", "database": { - "uri": "postgresql://${PG_USER}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/ondeck" + "managed": true }, "rules": [ "sqlc/db-prepare" diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 3548ce0df8..4b542d2f42 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -37,6 +37,7 @@ func Do(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int rootCmd.PersistentFlags().StringP("file", "f", "", "specify an alternate config file (default: sqlc.yaml)") rootCmd.PersistentFlags().BoolP("experimental", "x", false, "DEPRECATED: enable experimental features (default: false)") rootCmd.PersistentFlags().Bool("no-remote", false, "disable remote execution (default: false)") + rootCmd.PersistentFlags().Bool("remote", false, "enable remote execution (default: false)") rootCmd.PersistentFlags().Bool("no-database", false, "disable database connections (default: false)") rootCmd.AddCommand(checkCmd) @@ -136,17 +137,20 @@ var initCmd = &cobra.Command{ type Env struct { DryRun bool Debug opts.Debug + Remote bool NoRemote bool NoDatabase bool } func ParseEnv(c *cobra.Command) Env { dr := c.Flag("dry-run") + r := c.Flag("remote") nr := c.Flag("no-remote") nodb := c.Flag("no-database") return Env{ DryRun: dr != nil && dr.Changed, Debug: opts.DebugFromEnv(), + Remote: r != nil && nr.Value.String() == "true", NoRemote: nr != nil && nr.Value.String() == "true", NoDatabase: nodb != nil && nodb.Value.String() == "true", } diff --git a/internal/cmd/generate.go b/internal/cmd/generate.go index 40e1dcbcad..589295e74d 100644 --- a/internal/cmd/generate.go +++ b/internal/cmd/generate.go @@ -145,7 +145,7 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer return nil, err } - if conf.Cloud.Project != "" && !e.NoRemote { + if conf.Cloud.Project != "" && e.Remote && !e.NoRemote { return remoteGenerate(ctx, configPath, conf, dir, stderr) } diff --git a/internal/cmd/vet.go b/internal/cmd/vet.go index ed7b5937b7..49c166820a 100644 --- a/internal/cmd/vet.go +++ b/internal/cmd/vet.go @@ -415,10 +415,9 @@ func (c *checker) fetchDatabaseUri(ctx context.Context, s config.SQL) (string, f migrations = append(migrations, string(contents)) } - start := time.Now() resp, err := c.Client.CreateEphemeralDatabase(ctx, &pb.CreateEphemeralDatabaseRequest{ Engine: "postgresql", - Region: "sjc", + Region: quickdb.GetClosestRegion(), Migrations: migrations, }) if err != nil { @@ -432,7 +431,6 @@ func (c *checker) fetchDatabaseUri(ctx context.Context, s config.SQL) (string, f return err } - fmt.Println("createdb+template", time.Since(start)) return resp.Uri, cleanup, nil } diff --git a/internal/endtoend/vet_test.go b/internal/endtoend/vet_test.go index d453189e2c..43bf7e786e 100644 --- a/internal/endtoend/vet_test.go +++ b/internal/endtoend/vet_test.go @@ -51,12 +51,8 @@ func TestExamplesVet(t *testing.T) { path := filepath.Join(examples, tc) if tc != "kotlin" && tc != "python" { - if s, found := findSchema(t, filepath.Join(path, "postgresql")); found { - db, cleanup := sqltest.CreatePostgreSQLDatabase(t, tc, false, []string{s}) - defer db.Close() - defer cleanup() - } if s, found := findSchema(t, filepath.Join(path, "mysql")); found { + t.Skip("local") db, cleanup := sqltest.CreateMySQLDatabase(t, tc, []string{s}) defer db.Close() defer cleanup() From c20dc5ab248bfefb1a90efd17a91ad9a51ab98b4 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Thu, 21 Sep 2023 08:42:02 -0700 Subject: [PATCH 3/8] remove skip --- internal/endtoend/vet_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/endtoend/vet_test.go b/internal/endtoend/vet_test.go index 43bf7e786e..74f3ea9031 100644 --- a/internal/endtoend/vet_test.go +++ b/internal/endtoend/vet_test.go @@ -52,7 +52,6 @@ func TestExamplesVet(t *testing.T) { if tc != "kotlin" && tc != "python" { if s, found := findSchema(t, filepath.Join(path, "mysql")); found { - t.Skip("local") db, cleanup := sqltest.CreateMySQLDatabase(t, tc, []string{s}) defer db.Close() defer cleanup() From e27afd343264b52d8b38d9ffcaf734c9a3f875ab Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Thu, 21 Sep 2023 10:31:54 -0700 Subject: [PATCH 4/8] fix read file error --- internal/cmd/vet.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/cmd/vet.go b/internal/cmd/vet.go index 49c166820a..48fbc2a411 100644 --- a/internal/cmd/vet.go +++ b/internal/cmd/vet.go @@ -29,6 +29,7 @@ import ( "github.com/sqlc-dev/sqlc/internal/quickdb" pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1" "github.com/sqlc-dev/sqlc/internal/shfmt" + "github.com/sqlc-dev/sqlc/internal/sql/sqlpath" "github.com/sqlc-dev/sqlc/internal/vet" ) @@ -407,7 +408,11 @@ func (c *checker) fetchDatabaseUri(ctx context.Context, s config.SQL) (string, f } var migrations []string - for _, query := range s.Schema { + files, err := sqlpath.Glob(s.Schema) + if err != nil { + return "", cleanup, err + } + for _, query := range files { contents, err := os.ReadFile(query) if err != nil { return "", cleanup, fmt.Errorf("read file: %w", err) From e8812d83577fc4cdede2248370a86b3fe70d465d Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Thu, 21 Sep 2023 10:44:38 -0700 Subject: [PATCH 5/8] Add managed to JSON schema --- internal/config/v_one.json | 3 +++ internal/config/v_two.json | 3 +++ 2 files changed, 6 insertions(+) diff --git a/internal/config/v_one.json b/internal/config/v_one.json index 5d464a802a..329b012ff4 100644 --- a/internal/config/v_one.json +++ b/internal/config/v_one.json @@ -77,6 +77,9 @@ "properties": { "uri": { "type": "string" + }, + "managed": { + "type": "boolean" } } }, diff --git a/internal/config/v_two.json b/internal/config/v_two.json index bfbfaab676..65fdd7bb7e 100644 --- a/internal/config/v_two.json +++ b/internal/config/v_two.json @@ -77,6 +77,9 @@ "properties": { "uri": { "type": "string" + }, + "managed": { + "type": "boolean" } } }, From a3aa2310a50508336ce1f030edee5c657916969a Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Thu, 21 Sep 2023 15:03:17 -0700 Subject: [PATCH 6/8] Use different hostname for region discovery --- internal/quickdb/region.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/quickdb/region.go b/internal/quickdb/region.go index bc03e3b3f7..1900e33a14 100644 --- a/internal/quickdb/region.go +++ b/internal/quickdb/region.go @@ -10,9 +10,9 @@ var once sync.Once func GetClosestRegion() string { once.Do(func() { - resp, err := http.Get("https://debug.fly.dev") + resp, err := http.Get("https://find-closest-db-region.sqlc.dev") if err == nil { - region = resp.Header.Get("Fly-Region") + region = resp.Header.Get("Region") } }) return region From d12a8a656973a5d410f9c3246d7bac40e74e3f9a Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Thu, 21 Sep 2023 20:28:42 -0700 Subject: [PATCH 7/8] Add first pass at docs --- docs/howto/managed-databases.md | 63 +++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 docs/howto/managed-databases.md diff --git a/docs/howto/managed-databases.md b/docs/howto/managed-databases.md new file mode 100644 index 0000000000..8cbd192c8c --- /dev/null +++ b/docs/howto/managed-databases.md @@ -0,0 +1,63 @@ +# Managed databases + +*Added in v1.22.0* + +`sqlc` can create and maintain hosted databases for your project. These +databases can be used for linting queries. Right now, only PostgreSQL is +supported, with MySQL on the way. + +This feature is under active development. Beyond linting queries, managed +databases can be created per test suite or even per test, providing a real, +isolated PostgreSQL database for a test run, no cleanup required. + +Interested in trying out managed databases? Sign up [here](https://docs.google.com/forms/d/e/1FAIpQLSdxoMzJ7rKkBpuez-KyBcPNyckYV-5iMR--FRB7WnhvAmEvKg/viewform) or send us an email +at . + +## Configuring managed databases + +To configured a managed database, remove the `uri` key, replacing it with the +`managed` key set to `true`. Set the `project` key to your project ID, obtained +via the sqlc Dashboard. + +```yaml +version: '2' +cloud: + project: '' +sql: +- schema: schema.sql + queries: query.sql + engine: postgresql + database: + managed: true +``` + +## Authentication + +You'll also need to create an access token and make it available via the +`SQLC_AUTH_TOKEN` environment variable. + +```shell +export SQLC_AUTH_TOKEN=sqlc_xxxxxxxx +``` + +## Linting queries + +In managed mode, `sqlc vet` will create a database with the provided schema and +use that database when running lint rules. If you don't currently have any +rules, the [built-in sqlc/db-prepare] rule verifies each of your queries against +the database by creating a prepared statement. + +```yaml +version: '2' +cloud: + project: '' +sql: +- schema: schema.sql + queries: query.sql + engine: postgresql + database: + managed: true + rules: + - sqlc/db-prepare +``` + From ffaf99eb1db9291d53a81a3c6faf6205c7708290 Mon Sep 17 00:00:00 2001 From: Andrew Benton Date: Thu, 21 Sep 2023 23:11:50 -0700 Subject: [PATCH 8/8] docs: second pass at managed-databases.md --- docs/howto/managed-databases.md | 35 +++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/docs/howto/managed-databases.md b/docs/howto/managed-databases.md index 8cbd192c8c..70b62b2f91 100644 --- a/docs/howto/managed-databases.md +++ b/docs/howto/managed-databases.md @@ -3,21 +3,24 @@ *Added in v1.22.0* `sqlc` can create and maintain hosted databases for your project. These -databases can be used for linting queries. Right now, only PostgreSQL is -supported, with MySQL on the way. +databases are immediately useful for linting queries with [`sqlc vet`](vet.md) +if your lint rules require a connection to a running database. PostgreSQL +support is available today, with MySQL on the way. -This feature is under active development. Beyond linting queries, managed -databases can be created per test suite or even per test, providing a real, -isolated PostgreSQL database for a test run, no cleanup required. +This feature is under active development, and we're interested in supporting +other use-cases. Beyond linting queries, you can use sqlc managed databases +in your tests to quickly stand up a database per test suite or even per test, +providing a real, isolated database for a test run. No cleanup required. Interested in trying out managed databases? Sign up [here](https://docs.google.com/forms/d/e/1FAIpQLSdxoMzJ7rKkBpuez-KyBcPNyckYV-5iMR--FRB7WnhvAmEvKg/viewform) or send us an email at . ## Configuring managed databases -To configured a managed database, remove the `uri` key, replacing it with the -`managed` key set to `true`. Set the `project` key to your project ID, obtained -via the sqlc Dashboard. +To configure `sqlc` to use a managed database, remove the `uri` key from your +`database` configuration and replace it with the `managed` key set to `true`. +Set the `project` key in your `cloud` configuration to the value of your +project ID, obtained via the sqlc.dev Dashboard. ```yaml version: '2' @@ -33,8 +36,8 @@ sql: ## Authentication -You'll also need to create an access token and make it available via the -`SQLC_AUTH_TOKEN` environment variable. +`sqlc` expects to find a valid auth token in the value of the `SQLC_AUTH_TOKEN` +environment variable. You can create an auth token via the sqlc.dev Dashboard. ```shell export SQLC_AUTH_TOKEN=sqlc_xxxxxxxx @@ -42,10 +45,13 @@ export SQLC_AUTH_TOKEN=sqlc_xxxxxxxx ## Linting queries -In managed mode, `sqlc vet` will create a database with the provided schema and -use that database when running lint rules. If you don't currently have any -rules, the [built-in sqlc/db-prepare] rule verifies each of your queries against -the database by creating a prepared statement. +With managed databases configured, `sqlc vet` will create a database with your +package's schema and use that database when running lint rules that require a +database connection, e.g. any [rule relying on `EXPLAIN ...` output](vet.md#rules-using-explain-output). + +If you don't yet have any vet rules, the [built-in sqlc/db-prepare rule](vet.md#sqlc-db-prepare) +is a good place to start. It prepares each of your queries against the database +to ensure the query is valid. Here's a minimal working configuration: ```yaml version: '2' @@ -60,4 +66,3 @@ sql: rules: - sqlc/db-prepare ``` -