Skip to content

SQLite: Coerce jsonb columns to json before returning to Go code #3968

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions internal/codegen/golang/sqlite_type.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ func sqliteType(req *plugin.GenerateRequest, options *opts.Options, col *plugin.
}
return "sql.NullTime"

case "json", "jsonb":
return "json.RawMessage"

case "any":
return "interface{}"

Expand Down
4 changes: 4 additions & 0 deletions internal/compiler/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ type Compiler struct {
result *Result
analyzer analyzer.Analyzer
client dbmanager.Client
selector selector

schema []string
}
Expand All @@ -39,12 +40,15 @@ func NewCompiler(conf config.SQL, combo config.CombinedSettings) (*Compiler, err
case config.EngineSQLite:
c.parser = sqlite.NewParser()
c.catalog = sqlite.NewCatalog()
c.selector = newSQLiteSelector()
case config.EngineMySQL:
c.parser = dolphin.NewParser()
c.catalog = dolphin.NewCatalog()
c.selector = newDefaultSelector()
case config.EnginePostgreSQL:
c.parser = postgresql.NewParser()
c.catalog = postgresql.NewCatalog()
c.selector = newDefaultSelector()
if conf.Database != nil {
if conf.Analyzer.Database == nil || *conf.Analyzer.Database {
c.analyzer = analyzer.Cached(
Expand Down
5 changes: 5 additions & 0 deletions internal/compiler/expand.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,11 @@ func (c *Compiler) expandStmt(qc *QueryCatalog, raw *ast.RawStmt, node ast.Node)
if counts[cname] > 1 {
cname = tableName + "." + cname
}

// This is important for SQLite in particular which needs to
// wrap jsonb column values with `json(colname)` so they're in a
// publicly usable format (i.e. not jsonb).
cname = c.selector.ColumnExpr(cname, column)
cols = append(cols, cname)
}
}
Expand Down
46 changes: 46 additions & 0 deletions internal/compiler/selector.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package compiler

// selector is an interface used by a compiler for generating expressions for
// output columns in a `SELECT ...` or `RETURNING ...` statement.
//
// This interface is exclusively needed at the moment for SQLite, which must
// wrap output `jsonb` columns with a `json(column_name)` invocation so that a
// publicly consumable format (i.e. not jsonb) is returned.
type selector interface {
// ColumnExpr generates output to be used in a `SELECT ...` or `RETURNING
// ...` statement based on input column name and metadata.
ColumnExpr(name string, column *Column) string
}

// defaultSelector is a selector implementation that does the simpliest possible
// pass through when generating column expressions. Its use is suitable for all
// database engines not requiring additional customization.
type defaultSelector struct{}

func newDefaultSelector() *defaultSelector {
return &defaultSelector{}
}

func (s *defaultSelector) ColumnExpr(name string, column *Column) string {
return name
}

type sqliteSelector struct{}

func newSQLiteSelector() *sqliteSelector {
return &sqliteSelector{}
}

func (s *sqliteSelector) ColumnExpr(name string, column *Column) string {
// Under SQLite, neither json nor jsonb are real data types, and rather just
// of type blob, so database drivers just return whatever raw binary is
// stored as values. This is a problem for jsonb, which is considered an
// internal format to SQLite and no attempt should be made to parse it
// outside of the database itself. For jsonb columns in SQLite, wrap values
// in `json(col)` to coerce the internal binary format to JSON parsable by
// the user-space application.
if column.DataType == "jsonb" {
return "json(" + name + ")"
}
return name
}
35 changes: 35 additions & 0 deletions internal/compiler/selector_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package compiler

import "testing"

func TestSelector(t *testing.T) {
t.Parallel()

selectorExpectColumnExpr := func(t *testing.T, selector selector, expected, name string, column *Column) {
if actual := selector.ColumnExpr(name, column); expected != actual {
t.Errorf("Expected %v, got %v for data type %v", expected, actual, column.DataType)
}
}

t.Run("DefaultSelectorColumnExpr", func(t *testing.T) {
t.Parallel()

selector := newDefaultSelector()

selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "integer"})
selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "json"})
selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "jsonb"})
selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "text"})
})

t.Run("SQLiteSelectorColumnExpr", func(t *testing.T) {
t.Parallel()

selector := newSQLiteSelector()

selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "integer"})
selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "json"})
selectorExpectColumnExpr(t, selector, "json(my_column)", "my_column", &Column{DataType: "jsonb"})
selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "text"})
})
}
32 changes: 32 additions & 0 deletions internal/endtoend/testdata/jsonb/pgx/go/db.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 12 additions & 0 deletions internal/endtoend/testdata/jsonb/pgx/go/models.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

50 changes: 50 additions & 0 deletions internal/endtoend/testdata/jsonb/pgx/go/query.sql.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 15 additions & 0 deletions internal/endtoend/testdata/jsonb/pgx/query.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
-- name: InsertFoo :exec
INSERT INTO foo (
a,
b,
c,
d
) VALUES (
@a,
@b,
@c,
@d
) RETURNING *;

-- name: SelectFoo :exec
SELECT * FROM foo;
7 changes: 7 additions & 0 deletions internal/endtoend/testdata/jsonb/pgx/schema.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
CREATE TABLE foo (
a json not null,
b jsonb not null,
c json,
d jsonb
);

13 changes: 13 additions & 0 deletions internal/endtoend/testdata/jsonb/pgx/sqlc.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"version": "1",
"packages": [
{
"path": "go",
"engine": "postgresql",
"sql_package": "pgx/v5",
"name": "querytest",
"schema": "schema.sql",
"queries": "query.sql"
}
]
}
31 changes: 31 additions & 0 deletions internal/endtoend/testdata/jsonb/sqlite/go/db.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

16 changes: 16 additions & 0 deletions internal/endtoend/testdata/jsonb/sqlite/go/models.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

51 changes: 51 additions & 0 deletions internal/endtoend/testdata/jsonb/sqlite/go/query.sql.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 15 additions & 0 deletions internal/endtoend/testdata/jsonb/sqlite/query.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
-- name: InsertFoo :exec
INSERT INTO foo (
a,
b,
c,
d
) VALUES (
@a,
@b,
@c,
@d
) RETURNING *;

-- name: SelectFoo :exec
SELECT * FROM foo;
7 changes: 7 additions & 0 deletions internal/endtoend/testdata/jsonb/sqlite/schema.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
CREATE TABLE foo (
a json not null,
b jsonb not null,
c json,
d jsonb
);

Loading
Loading