diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6da2dbd1..970d2771 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,8 +8,8 @@ jobs: strategy: fail-fast: false matrix: - node: ['12', '14', '16', '18', '20', '21'] - postgres: ['12', '13', '14', '15', '16'] + node: ['12', '14', '16', '18', '20', '21', '22', '23', '24'] + postgres: ['12', '13', '14', '15', '16', '17'] runs-on: ubuntu-latest services: postgres: @@ -25,10 +25,10 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: | date - sudo apt purge postgresql-14 + sudo apt purge postgresql-16 sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo apt-get update @@ -48,7 +48,7 @@ jobs: - uses: denoland/setup-deno@v1 with: deno-version: v1.x - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version: ${{ matrix.node }} - run: npm test diff --git a/README.md b/README.md index da002cca..b04ac21c 100644 --- a/README.md +++ b/README.md @@ -270,7 +270,7 @@ const users = [ ] await sql` - update users set name = update_data.name, (age = update_data.age)::int + update users set name = update_data.name, age = (update_data.age)::int from (values ${sql(users)}) as update_data (id, name, age) where users.id = (update_data.id)::int returning users.id, users.name, users.age @@ -290,7 +290,7 @@ const users = await sql` or ```js -const [{ a, b, c }] => await sql` +const [{ a, b, c }] = await sql` select * from (values ${ sql(['a', 'b', 'c']) }) as x(a, b, c) @@ -342,6 +342,27 @@ select * from users select * from users where user_id = $1 ``` +### Dynamic ordering + +```js +const id = 1 +const order = { + username: 'asc' + created_at: 'desc' +} +await sql` + select + * + from ticket + where account = ${ id } + order by ${ + Object.entries(order).flatMap(([column, order], i) => + [i ? sql`,` : sql``, sql`${ sql(column) } ${ order === 'desc' ? sql`desc` : sql`asc` }`] + ) + } +` +``` + ### SQL functions Using keywords or calling functions dynamically is also possible by using ``` sql`` ``` fragments. ```js @@ -537,7 +558,7 @@ for await (const chunk of readableStream) { } ``` -> **NOTE** This is a low-level API which does not provide any type safety. To make this work, you must match your [`copy query` parameters](https://www.postgresql.org/docs/14/sql-copy.html) correctly to your [Node.js stream read or write](https://nodejs.org/api/stream.html) code. Ensure [Node.js stream backpressure](https://nodejs.org/en/docs/guides/backpressuring-in-streams/) is handled correctly to avoid memory exhaustion. +> **NOTE** This is a low-level API which does not provide any type safety. To make this work, you must match your [`copy query` parameters](https://www.postgresql.org/docs/14/sql-copy.html) correctly to your [Node.js stream read or write](https://nodejs.org/api/stream.html) code. Ensure [Node.js stream backpressure](https://nodejs.org/en/learn/modules/backpressuring-in-streams) is handled correctly to avoid memory exhaustion. ### Canceling Queries in Progress @@ -568,6 +589,8 @@ If you know what you're doing, you can use `unsafe` to pass any string you'd lik sql.unsafe('select ' + danger + ' from users where id = ' + dragons) ``` +By default, `sql.unsafe` assumes the `query` string is sufficiently dynamic that prepared statements do not make sense, and so defaults them to off. If you'd like to re-enable prepared statements, you can pass `{ prepare: true }`. + You can also nest `sql.unsafe` within a safe `sql` expression. This is useful if only part of your fraction has unsafe elements. ```js @@ -917,7 +940,7 @@ The `Result` Array returned from queries is a custom array allowing for easy des ### .count -The `count` property is the number of affected rows returned by the database. This is usefull for insert, update and delete operations to know the number of rows since .length will be 0 in these cases if not using `RETURNING ...`. +The `count` property is the number of affected rows returned by the database. This is useful for insert, update and delete operations to know the number of rows since .length will be 0 in these cases if not using `RETURNING ...`. ### .command @@ -992,7 +1015,7 @@ const sql = postgres('postgres://username:password@host:port/database', { }) ``` -Note that `max_lifetime = 60 * (30 + Math.random() * 30)` by default. This resolves to an interval between 45 and 90 minutes to optimize for the benefits of prepared statements **and** working nicely with Linux's OOM killer. +Note that `max_lifetime = 60 * (30 + Math.random() * 30)` by default. This resolves to an interval between 30 and 60 minutes to optimize for the benefits of prepared statements **and** working nicely with Linux's OOM killer. ### Dynamic passwords @@ -1103,10 +1126,10 @@ export default async fetch(req: Request, env: Env, ctx: ExecutionContext) { } ``` -In `wrangler.toml` you will need to enable `node_compat` to allow Postgres.js to operate in the Workers environment: +In `wrangler.toml` you will need to enable the `nodejs_compat` compatibility flag to allow Postgres.js to operate in the Workers environment: ```toml -node_compat = true # required for database drivers to function +compatibility_flags = ["nodejs_compat"] ``` ### Auto fetching of array types @@ -1125,20 +1148,25 @@ It is also possible to connect to the database without a connection string or an const sql = postgres() ``` -| Option | Environment Variables | -| ----------------- | ------------------------ | -| `host` | `PGHOST` | -| `port` | `PGPORT` | -| `database` | `PGDATABASE` | -| `username` | `PGUSERNAME` or `PGUSER` | -| `password` | `PGPASSWORD` | -| `idle_timeout` | `PGIDLE_TIMEOUT` | -| `connect_timeout` | `PGCONNECT_TIMEOUT` | +| Option | Environment Variables | +| ------------------ | ------------------------ | +| `host` | `PGHOST` | +| `port` | `PGPORT` | +| `database` | `PGDATABASE` | +| `username` | `PGUSERNAME` or `PGUSER` | +| `password` | `PGPASSWORD` | +| `application_name` | `PGAPPNAME` | +| `idle_timeout` | `PGIDLE_TIMEOUT` | +| `connect_timeout` | `PGCONNECT_TIMEOUT` | ### Prepared statements Prepared statements will automatically be created for any queries where it can be inferred that the query is static. This can be disabled by using the `prepare: false` option. For instance — this is useful when [using PGBouncer in `transaction mode`](https://github.com/porsager/postgres/issues/93#issuecomment-656290493). +**update**: [since 1.21.0](https://www.pgbouncer.org/2023/10/pgbouncer-1-21-0) +PGBouncer supports protocol-level named prepared statements when [configured +properly](https://www.pgbouncer.org/config.html#max_prepared_statements) + ## Custom Types You can add ergonomic support for custom types, or simply use `sql.typed(value, type)` inline, where type is the PostgreSQL `oid` for the type and the correctly serialized string. _(`oid` values for types can be found in the `pg_catalog.pg_type` table.)_ @@ -1298,8 +1326,8 @@ This error is thrown if the user has called [`sql.end()`](#teardown--cleanup) an This error is thrown for any queries that were pending when the timeout to [`sql.end({ timeout: X })`](#teardown--cleanup) was reached. -##### CONNECTION_CONNECT_TIMEOUT -> write CONNECTION_CONNECT_TIMEOUT host:port +##### CONNECT_TIMEOUT +> write CONNECT_TIMEOUT host:port This error is thrown if the startup phase of the connection (tcp, protocol negotiation, and auth) took more than the default 30 seconds or what was specified using `connect_timeout` or `PGCONNECT_TIMEOUT`. diff --git a/cf/src/connection.js b/cf/src/connection.js index f06a5f8b..203af80d 100644 --- a/cf/src/connection.js +++ b/cf/src/connection.js @@ -295,7 +295,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose if (incomings) { incomings.push(x) remaining -= x.length - if (remaining >= 0) + if (remaining > 0) return } @@ -387,7 +387,13 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } function queryError(query, err) { - Object.defineProperties(err, { + if (query.reserve) + return query.reject(err) + + if (!err || typeof err !== 'object') + err = new Error(err) + + 'query' in err || 'parameters' in err || Object.defineProperties(err, { stack: { value: err.stack + query.origin.replace(/.*\n/, '\n'), enumerable: options.debug }, query: { value: query.string, enumerable: options.debug }, parameters: { value: query.parameters, enumerable: options.debug }, @@ -431,10 +437,8 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose lifeTimer.cancel() connectTimer.cancel() - if (socket.encrypted) { - socket.removeAllListeners() - socket = null - } + socket.removeAllListeners() + socket = null if (initial) return reconnect() @@ -535,11 +539,14 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose return terminate() } - if (needsTypes) + if (needsTypes) { + initial.reserve && (initial = null) return fetchArrayTypes() + } - execute(initial) - options.shared.retries = retries = initial = 0 + initial && !initial.reserve && execute(initial) + options.shared.retries = retries = 0 + initial = null return } @@ -789,7 +796,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose const error = Errors.postgres(parseError(x)) query && query.retried ? errored(query.retried) - : query && retryRoutines.has(error.routine) + : query && query.prepared && retryRoutines.has(error.routine) ? retry(query, error) : errored(error) } diff --git a/cf/src/index.js b/cf/src/index.js index d24e9f9c..3ffb7e65 100644 --- a/cf/src/index.js +++ b/cf/src/index.js @@ -205,9 +205,10 @@ function Postgres(a, b) { const queue = Queue() const c = open.length ? open.shift() - : await new Promise(r => { - queries.push({ reserve: r }) - closed.length && connect(closed.shift()) + : await new Promise((resolve, reject) => { + const query = { reserve: resolve, reject } + queries.push(query) + closed.length && connect(closed.shift(), query) }) move(c, reserved) @@ -481,7 +482,7 @@ function parseOptions(a, b) { {} ), connection : { - application_name: 'postgres.js', + application_name: env.PGAPPNAME || 'postgres.js', ...o.connection, ...Object.entries(query).reduce((acc, [k, v]) => (k in defaults || (acc[k] = v), acc), {}) }, diff --git a/cf/src/subscribe.js b/cf/src/subscribe.js index 1ab8b0be..8716100e 100644 --- a/cf/src/subscribe.js +++ b/cf/src/subscribe.js @@ -48,7 +48,7 @@ export default function Subscribe(postgres, options) { return subscribe - async function subscribe(event, fn, onsubscribe = noop) { + async function subscribe(event, fn, onsubscribe = noop, onerror = noop) { event = parseEvent(event) if (!connection) @@ -67,6 +67,7 @@ export default function Subscribe(postgres, options) { return connection.then(x => { connected(x) onsubscribe() + stream && stream.on('error', onerror) return { unsubscribe, state, sql } }) } @@ -104,14 +105,16 @@ export default function Subscribe(postgres, options) { return { stream, state: xs.state } function error(e) { - console.error('Unexpected error during logical streaming - reconnecting', e) + console.error('Unexpected error during logical streaming - reconnecting', e) // eslint-disable-line } function data(x) { - if (x[0] === 0x77) + if (x[0] === 0x77) { parse(x.subarray(25), state, sql.options.parsers, handle, options.transform) - else if (x[0] === 0x6b && x[17]) + } else if (x[0] === 0x6b && x[17]) { + state.lsn = x.subarray(1, 9) pong() + } } function handle(a, b) { diff --git a/cjs/src/connection.js b/cjs/src/connection.js index b295958a..589d3638 100644 --- a/cjs/src/connection.js +++ b/cjs/src/connection.js @@ -293,7 +293,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose if (incomings) { incomings.push(x) remaining -= x.length - if (remaining >= 0) + if (remaining > 0) return } @@ -385,7 +385,13 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } function queryError(query, err) { - Object.defineProperties(err, { + if (query.reserve) + return query.reject(err) + + if (!err || typeof err !== 'object') + err = new Error(err) + + 'query' in err || 'parameters' in err || Object.defineProperties(err, { stack: { value: err.stack + query.origin.replace(/.*\n/, '\n'), enumerable: options.debug }, query: { value: query.string, enumerable: options.debug }, parameters: { value: query.parameters, enumerable: options.debug }, @@ -429,10 +435,8 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose lifeTimer.cancel() connectTimer.cancel() - if (socket.encrypted) { - socket.removeAllListeners() - socket = null - } + socket.removeAllListeners() + socket = null if (initial) return reconnect() @@ -533,11 +537,14 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose return terminate() } - if (needsTypes) + if (needsTypes) { + initial.reserve && (initial = null) return fetchArrayTypes() + } - execute(initial) - options.shared.retries = retries = initial = 0 + initial && !initial.reserve && execute(initial) + options.shared.retries = retries = 0 + initial = null return } @@ -787,7 +794,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose const error = Errors.postgres(parseError(x)) query && query.retried ? errored(query.retried) - : query && retryRoutines.has(error.routine) + : query && query.prepared && retryRoutines.has(error.routine) ? retry(query, error) : errored(error) } diff --git a/cjs/src/index.js b/cjs/src/index.js index 40ac2c18..baf7e60a 100644 --- a/cjs/src/index.js +++ b/cjs/src/index.js @@ -204,9 +204,10 @@ function Postgres(a, b) { const queue = Queue() const c = open.length ? open.shift() - : await new Promise(r => { - queries.push({ reserve: r }) - closed.length && connect(closed.shift()) + : await new Promise((resolve, reject) => { + const query = { reserve: resolve, reject } + queries.push(query) + closed.length && connect(closed.shift(), query) }) move(c, reserved) @@ -480,7 +481,7 @@ function parseOptions(a, b) { {} ), connection : { - application_name: 'postgres.js', + application_name: env.PGAPPNAME || 'postgres.js', ...o.connection, ...Object.entries(query).reduce((acc, [k, v]) => (k in defaults || (acc[k] = v), acc), {}) }, diff --git a/cjs/src/subscribe.js b/cjs/src/subscribe.js index 34d99e9f..6aaa8962 100644 --- a/cjs/src/subscribe.js +++ b/cjs/src/subscribe.js @@ -47,7 +47,7 @@ module.exports = Subscribe;function Subscribe(postgres, options) { return subscribe - async function subscribe(event, fn, onsubscribe = noop) { + async function subscribe(event, fn, onsubscribe = noop, onerror = noop) { event = parseEvent(event) if (!connection) @@ -66,6 +66,7 @@ module.exports = Subscribe;function Subscribe(postgres, options) { return connection.then(x => { connected(x) onsubscribe() + stream && stream.on('error', onerror) return { unsubscribe, state, sql } }) } @@ -103,14 +104,16 @@ module.exports = Subscribe;function Subscribe(postgres, options) { return { stream, state: xs.state } function error(e) { - console.error('Unexpected error during logical streaming - reconnecting', e) + console.error('Unexpected error during logical streaming - reconnecting', e) // eslint-disable-line } function data(x) { - if (x[0] === 0x77) + if (x[0] === 0x77) { parse(x.subarray(25), state, sql.options.parsers, handle, options.transform) - else if (x[0] === 0x6b && x[17]) + } else if (x[0] === 0x6b && x[17]) { + state.lsn = x.subarray(1, 9) pong() + } } function handle(a, b) { diff --git a/cjs/tests/index.js b/cjs/tests/index.js index ef70c4ab..ec5222f7 100644 --- a/cjs/tests/index.js +++ b/cjs/tests/index.js @@ -429,6 +429,30 @@ t('Reconnect using SSL', { timeout: 2 }, async() => { return [1, (await sql`select 1 as x`)[0].x] }) +t('Proper handling of non object Errors', async() => { + const sql = postgres({ socket: () => { throw 'wat' } }) // eslint-disable-line + + return [ + 'wat', await sql`select 1 as x`.catch(e => e.message) + ] +}) + +t('Proper handling of null Errors', async() => { + const sql = postgres({ socket: () => { throw null } }) // eslint-disable-line + + return [ + 'null', await sql`select 1 as x`.catch(e => e.message) + ] +}) + +t('Ensure reserve on connection throws proper error', async() => { + const sql = postgres({ socket: () => { throw 'wat' }, idle_timeout }) // eslint-disable-line + + return [ + 'wat', await sql.reserve().catch(e => e) + ] +}) + t('Login without password', async() => { return [true, (await postgres({ ...options, ...login })`select true as x`)[0].x] }) @@ -1789,6 +1813,32 @@ t('Recreate prepared statements on RevalidateCachedQuery error', async() => { ] }) +t('Properly throws routine error on not prepared statements', async() => { + await sql`create table x (x text[])` + const { routine } = await sql.unsafe(` + insert into x(x) values (('a', 'b')) + `).catch(e => e) + + return ['transformAssignedExpr', routine, await sql`drop table x`] +}) + +t('Properly throws routine error on not prepared statements in transaction', async() => { + const { routine } = await sql.begin(sql => [ + sql`create table x (x text[])`, + sql`insert into x(x) values (('a', 'b'))` + ]).catch(e => e) + + return ['transformAssignedExpr', routine] +}) + +t('Properly throws routine error on not prepared statements using file', async() => { + const { routine } = await sql.unsafe(` + create table x (x text[]); + insert into x(x) values (('a', 'b')); + `, { prepare: true }).catch(e => e) + + return ['transformAssignedExpr', routine] +}) t('Catches connection config errors', async() => { const sql = postgres({ ...options, user: { toString: () => { throw new Error('wat') } }, database: 'prut' }) @@ -2543,3 +2593,24 @@ t('reserve connection', async() => { xs.map(x => x.x).join('') ] }) + +t('arrays in reserved connection', async() => { + const reserved = await sql.reserve() + const [{ x }] = await reserved`select array[1, 2, 3] as x` + reserved.release() + + return [ + '123', + x.join('') + ] +}) + +t('Ensure reserve on query throws proper error', async() => { + const sql = postgres({ idle_timeout }) // eslint-disable-line + const reserved = await sql.reserve() + const [{ x }] = await reserved`select 'wat' as x` + + return [ + 'wat', x, reserved.release() + ] +}) diff --git a/deno/README.md b/deno/README.md index 0fc569bb..b6ec85b7 100644 --- a/deno/README.md +++ b/deno/README.md @@ -266,7 +266,7 @@ const users = [ ] await sql` - update users set name = update_data.name, (age = update_data.age)::int + update users set name = update_data.name, age = (update_data.age)::int from (values ${sql(users)}) as update_data (id, name, age) where users.id = (update_data.id)::int returning users.id, users.name, users.age @@ -286,7 +286,7 @@ const users = await sql` or ```js -const [{ a, b, c }] => await sql` +const [{ a, b, c }] = await sql` select * from (values ${ sql(['a', 'b', 'c']) }) as x(a, b, c) @@ -338,6 +338,27 @@ select * from users select * from users where user_id = $1 ``` +### Dynamic ordering + +```js +const id = 1 +const order = { + username: 'asc' + created_at: 'desc' +} +await sql` + select + * + from ticket + where account = ${ id } + order by ${ + Object.entries(order).flatMap(([column, order], i) => + [i ? sql`,` : sql``, sql`${ sql(column) } ${ order === 'desc' ? sql`desc` : sql`asc` }`] + ) + } +` +``` + ### SQL functions Using keywords or calling functions dynamically is also possible by using ``` sql`` ``` fragments. ```js @@ -533,7 +554,7 @@ for await (const chunk of readableStream) { } ``` -> **NOTE** This is a low-level API which does not provide any type safety. To make this work, you must match your [`copy query` parameters](https://www.postgresql.org/docs/14/sql-copy.html) correctly to your [Node.js stream read or write](https://nodejs.org/api/stream.html) code. Ensure [Node.js stream backpressure](https://nodejs.org/en/docs/guides/backpressuring-in-streams/) is handled correctly to avoid memory exhaustion. +> **NOTE** This is a low-level API which does not provide any type safety. To make this work, you must match your [`copy query` parameters](https://www.postgresql.org/docs/14/sql-copy.html) correctly to your [Node.js stream read or write](https://nodejs.org/api/stream.html) code. Ensure [Node.js stream backpressure](https://nodejs.org/en/learn/modules/backpressuring-in-streams) is handled correctly to avoid memory exhaustion. ### Canceling Queries in Progress @@ -564,6 +585,8 @@ If you know what you're doing, you can use `unsafe` to pass any string you'd lik sql.unsafe('select ' + danger + ' from users where id = ' + dragons) ``` +By default, `sql.unsafe` assumes the `query` string is sufficiently dynamic that prepared statements do not make sense, and so defaults them to off. If you'd like to re-enable prepared statements, you can pass `{ prepare: true }`. + You can also nest `sql.unsafe` within a safe `sql` expression. This is useful if only part of your fraction has unsafe elements. ```js @@ -913,7 +936,7 @@ The `Result` Array returned from queries is a custom array allowing for easy des ### .count -The `count` property is the number of affected rows returned by the database. This is usefull for insert, update and delete operations to know the number of rows since .length will be 0 in these cases if not using `RETURNING ...`. +The `count` property is the number of affected rows returned by the database. This is useful for insert, update and delete operations to know the number of rows since .length will be 0 in these cases if not using `RETURNING ...`. ### .command @@ -988,7 +1011,7 @@ const sql = postgres('postgres://username:password@host:port/database', { }) ``` -Note that `max_lifetime = 60 * (30 + Math.random() * 30)` by default. This resolves to an interval between 45 and 90 minutes to optimize for the benefits of prepared statements **and** working nicely with Linux's OOM killer. +Note that `max_lifetime = 60 * (30 + Math.random() * 30)` by default. This resolves to an interval between 30 and 60 minutes to optimize for the benefits of prepared statements **and** working nicely with Linux's OOM killer. ### Dynamic passwords @@ -1099,10 +1122,10 @@ export default async fetch(req: Request, env: Env, ctx: ExecutionContext) { } ``` -In `wrangler.toml` you will need to enable `node_compat` to allow Postgres.js to operate in the Workers environment: +In `wrangler.toml` you will need to enable the `nodejs_compat` compatibility flag to allow Postgres.js to operate in the Workers environment: ```toml -node_compat = true # required for database drivers to function +compatibility_flags = ["nodejs_compat"] ``` ### Auto fetching of array types @@ -1121,20 +1144,25 @@ It is also possible to connect to the database without a connection string or an const sql = postgres() ``` -| Option | Environment Variables | -| ----------------- | ------------------------ | -| `host` | `PGHOST` | -| `port` | `PGPORT` | -| `database` | `PGDATABASE` | -| `username` | `PGUSERNAME` or `PGUSER` | -| `password` | `PGPASSWORD` | -| `idle_timeout` | `PGIDLE_TIMEOUT` | -| `connect_timeout` | `PGCONNECT_TIMEOUT` | +| Option | Environment Variables | +| ------------------ | ------------------------ | +| `host` | `PGHOST` | +| `port` | `PGPORT` | +| `database` | `PGDATABASE` | +| `username` | `PGUSERNAME` or `PGUSER` | +| `password` | `PGPASSWORD` | +| `application_name` | `PGAPPNAME` | +| `idle_timeout` | `PGIDLE_TIMEOUT` | +| `connect_timeout` | `PGCONNECT_TIMEOUT` | ### Prepared statements Prepared statements will automatically be created for any queries where it can be inferred that the query is static. This can be disabled by using the `prepare: false` option. For instance — this is useful when [using PGBouncer in `transaction mode`](https://github.com/porsager/postgres/issues/93#issuecomment-656290493). +**update**: [since 1.21.0](https://www.pgbouncer.org/2023/10/pgbouncer-1-21-0) +PGBouncer supports protocol-level named prepared statements when [configured +properly](https://www.pgbouncer.org/config.html#max_prepared_statements) + ## Custom Types You can add ergonomic support for custom types, or simply use `sql.typed(value, type)` inline, where type is the PostgreSQL `oid` for the type and the correctly serialized string. _(`oid` values for types can be found in the `pg_catalog.pg_type` table.)_ @@ -1294,8 +1322,8 @@ This error is thrown if the user has called [`sql.end()`](#teardown--cleanup) an This error is thrown for any queries that were pending when the timeout to [`sql.end({ timeout: X })`](#teardown--cleanup) was reached. -##### CONNECTION_CONNECT_TIMEOUT -> write CONNECTION_CONNECT_TIMEOUT host:port +##### CONNECT_TIMEOUT +> write CONNECT_TIMEOUT host:port This error is thrown if the startup phase of the connection (tcp, protocol negotiation, and auth) took more than the default 30 seconds or what was specified using `connect_timeout` or `PGCONNECT_TIMEOUT`. diff --git a/deno/src/connection.js b/deno/src/connection.js index bc4d231c..a3f43c48 100644 --- a/deno/src/connection.js +++ b/deno/src/connection.js @@ -296,7 +296,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose if (incomings) { incomings.push(x) remaining -= x.length - if (remaining >= 0) + if (remaining > 0) return } @@ -388,7 +388,13 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } function queryError(query, err) { - Object.defineProperties(err, { + if (query.reserve) + return query.reject(err) + + if (!err || typeof err !== 'object') + err = new Error(err) + + 'query' in err || 'parameters' in err || Object.defineProperties(err, { stack: { value: err.stack + query.origin.replace(/.*\n/, '\n'), enumerable: options.debug }, query: { value: query.string, enumerable: options.debug }, parameters: { value: query.parameters, enumerable: options.debug }, @@ -432,10 +438,8 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose lifeTimer.cancel() connectTimer.cancel() - if (socket.encrypted) { - socket.removeAllListeners() - socket = null - } + socket.removeAllListeners() + socket = null if (initial) return reconnect() @@ -536,11 +540,14 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose return terminate() } - if (needsTypes) + if (needsTypes) { + initial.reserve && (initial = null) return fetchArrayTypes() + } - execute(initial) - options.shared.retries = retries = initial = 0 + initial && !initial.reserve && execute(initial) + options.shared.retries = retries = 0 + initial = null return } @@ -790,7 +797,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose const error = Errors.postgres(parseError(x)) query && query.retried ? errored(query.retried) - : query && retryRoutines.has(error.routine) + : query && query.prepared && retryRoutines.has(error.routine) ? retry(query, error) : errored(error) } diff --git a/deno/src/index.js b/deno/src/index.js index 3bbdf2ba..aa7a920f 100644 --- a/deno/src/index.js +++ b/deno/src/index.js @@ -205,9 +205,10 @@ function Postgres(a, b) { const queue = Queue() const c = open.length ? open.shift() - : await new Promise(r => { - queries.push({ reserve: r }) - closed.length && connect(closed.shift()) + : await new Promise((resolve, reject) => { + const query = { reserve: resolve, reject } + queries.push(query) + closed.length && connect(closed.shift(), query) }) move(c, reserved) @@ -481,8 +482,8 @@ function parseOptions(a, b) { {} ), connection : { - application_name: 'postgres.js', ...o.connection, + application_name: o.connection?.application_name ?? env.PGAPPNAME ?? 'postgres.js', ...Object.entries(query).reduce((acc, [k, v]) => (k in defaults || (acc[k] = v), acc), {}) }, types : o.types || {}, diff --git a/deno/src/subscribe.js b/deno/src/subscribe.js index dbb9b971..b20efb96 100644 --- a/deno/src/subscribe.js +++ b/deno/src/subscribe.js @@ -48,7 +48,7 @@ export default function Subscribe(postgres, options) { return subscribe - async function subscribe(event, fn, onsubscribe = noop) { + async function subscribe(event, fn, onsubscribe = noop, onerror = noop) { event = parseEvent(event) if (!connection) @@ -67,6 +67,7 @@ export default function Subscribe(postgres, options) { return connection.then(x => { connected(x) onsubscribe() + stream && stream.on('error', onerror) return { unsubscribe, state, sql } }) } @@ -104,14 +105,16 @@ export default function Subscribe(postgres, options) { return { stream, state: xs.state } function error(e) { - console.error('Unexpected error during logical streaming - reconnecting', e) + console.error('Unexpected error during logical streaming - reconnecting', e) // eslint-disable-line } function data(x) { - if (x[0] === 0x77) + if (x[0] === 0x77) { parse(x.subarray(25), state, sql.options.parsers, handle, options.transform) - else if (x[0] === 0x6b && x[17]) + } else if (x[0] === 0x6b && x[17]) { + state.lsn = x.subarray(1, 9) pong() + } } function handle(a, b) { diff --git a/deno/tests/index.js b/deno/tests/index.js index dc78c2c8..adedf1e0 100644 --- a/deno/tests/index.js +++ b/deno/tests/index.js @@ -431,6 +431,30 @@ t('Reconnect using SSL', { timeout: 2 }, async() => { return [1, (await sql`select 1 as x`)[0].x] }) +t('Proper handling of non object Errors', async() => { + const sql = postgres({ socket: () => { throw 'wat' } }) // eslint-disable-line + + return [ + 'wat', await sql`select 1 as x`.catch(e => e.message) + ] +}) + +t('Proper handling of null Errors', async() => { + const sql = postgres({ socket: () => { throw null } }) // eslint-disable-line + + return [ + 'null', await sql`select 1 as x`.catch(e => e.message) + ] +}) + +t('Ensure reserve on connection throws proper error', async() => { + const sql = postgres({ socket: () => { throw 'wat' }, idle_timeout }) // eslint-disable-line + + return [ + 'wat', await sql.reserve().catch(e => e) + ] +}) + t('Login without password', async() => { return [true, (await postgres({ ...options, ...login })`select true as x`)[0].x] }) @@ -1791,6 +1815,32 @@ t('Recreate prepared statements on RevalidateCachedQuery error', async() => { ] }) +t('Properly throws routine error on not prepared statements', async() => { + await sql`create table x (x text[])` + const { routine } = await sql.unsafe(` + insert into x(x) values (('a', 'b')) + `).catch(e => e) + + return ['transformAssignedExpr', routine, await sql`drop table x`] +}) + +t('Properly throws routine error on not prepared statements in transaction', async() => { + const { routine } = await sql.begin(sql => [ + sql`create table x (x text[])`, + sql`insert into x(x) values (('a', 'b'))` + ]).catch(e => e) + + return ['transformAssignedExpr', routine] +}) + +t('Properly throws routine error on not prepared statements using file', async() => { + const { routine } = await sql.unsafe(` + create table x (x text[]); + insert into x(x) values (('a', 'b')); + `, { prepare: true }).catch(e => e) + + return ['transformAssignedExpr', routine] +}) t('Catches connection config errors', async() => { const sql = postgres({ ...options, user: { toString: () => { throw new Error('wat') } }, database: 'prut' }) @@ -2546,4 +2596,25 @@ t('reserve connection', async() => { ] }) -;window.addEventListener("unload", () => Deno.exit(process.exitCode)) \ No newline at end of file +t('arrays in reserved connection', async() => { + const reserved = await sql.reserve() + const [{ x }] = await reserved`select array[1, 2, 3] as x` + reserved.release() + + return [ + '123', + x.join('') + ] +}) + +t('Ensure reserve on query throws proper error', async() => { + const sql = postgres({ idle_timeout }) // eslint-disable-line + const reserved = await sql.reserve() + const [{ x }] = await reserved`select 'wat' as x` + + return [ + 'wat', x, reserved.release() + ] +}) + +;globalThis.addEventListener("unload", () => Deno.exit(process.exitCode)) \ No newline at end of file diff --git a/deno/types/index.d.ts b/deno/types/index.d.ts index 6f96fe97..44a07af0 100644 --- a/deno/types/index.d.ts +++ b/deno/types/index.d.ts @@ -458,7 +458,8 @@ declare namespace postgres { | 'NOT_TAGGED_CALL' | 'UNDEFINED_VALUE' | 'MAX_PARAMETERS_EXCEEDED' - | 'SASL_SIGNATURE_MISMATCH'; + | 'SASL_SIGNATURE_MISMATCH' + | 'UNSAFE_TRANSACTION'; message: string; } @@ -601,6 +602,7 @@ declare namespace postgres { type RowList = T & Iterable> & ResultQueryMeta; interface PendingQueryModifiers { + simple(): this; readable(): Promise; writable(): Promise; @@ -692,7 +694,7 @@ declare namespace postgres { listen(channel: string, onnotify: (value: string) => void, onlisten?: (() => void) | undefined): ListenRequest; notify(channel: string, payload: string): PendingRequest; - subscribe(event: string, cb: (row: Row | null, info: ReplicationEvent) => void, onsubscribe?: (() => void) | undefined): Promise; + subscribe(event: string, cb: (row: Row | null, info: ReplicationEvent) => void, onsubscribe?: (() => void), onerror?: (() => any)): Promise; largeObject(oid?: number | undefined, /** @default 0x00020000 | 0x00040000 */ mode?: number | undefined): Promise; diff --git a/package.json b/package.json index 34802d6c..65157609 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "postgres", - "version": "3.4.2", + "version": "3.4.7", "description": "Fastest full featured PostgreSQL client for Node.js", "type": "module", "module": "src/index.js", @@ -8,7 +8,7 @@ "exports": { "types": "./types/index.d.ts", "bun": "./src/index.js", - "worker": "./cf/src/index.js", + "workerd": "./cf/src/index.js", "import": "./src/index.js", "default": "./cjs/src/index.js" }, @@ -25,7 +25,7 @@ "test": "npm run test:esm && npm run test:cjs && npm run test:deno", "test:esm": "node tests/index.js", "test:cjs": "npm run build:cjs && cd cjs/tests && node index.js && cd ../../", - "test:deno": "npm run build:deno && cd deno/tests && deno run --unstable --allow-all --unsafely-ignore-certificate-errors index.js && cd ../../", + "test:deno": "npm run build:deno && cd deno/tests && deno run --no-lock --allow-all --unsafely-ignore-certificate-errors index.js && cd ../../", "lint": "eslint src && eslint tests", "prepare": "npm run build", "prepublishOnly": "npm run lint" diff --git a/src/connection.js b/src/connection.js index a6825105..c3f554aa 100644 --- a/src/connection.js +++ b/src/connection.js @@ -293,7 +293,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose if (incomings) { incomings.push(x) remaining -= x.length - if (remaining >= 0) + if (remaining > 0) return } @@ -385,7 +385,13 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } function queryError(query, err) { - Object.defineProperties(err, { + if (query.reserve) + return query.reject(err) + + if (!err || typeof err !== 'object') + err = new Error(err) + + 'query' in err || 'parameters' in err || Object.defineProperties(err, { stack: { value: err.stack + query.origin.replace(/.*\n/, '\n'), enumerable: options.debug }, query: { value: query.string, enumerable: options.debug }, parameters: { value: query.parameters, enumerable: options.debug }, @@ -429,10 +435,8 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose lifeTimer.cancel() connectTimer.cancel() - if (socket.encrypted) { - socket.removeAllListeners() - socket = null - } + socket.removeAllListeners() + socket = null if (initial) return reconnect() @@ -533,11 +537,14 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose return terminate() } - if (needsTypes) + if (needsTypes) { + initial.reserve && (initial = null) return fetchArrayTypes() + } - execute(initial) - options.shared.retries = retries = initial = 0 + initial && !initial.reserve && execute(initial) + options.shared.retries = retries = 0 + initial = null return } @@ -787,7 +794,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose const error = Errors.postgres(parseError(x)) query && query.retried ? errored(query.retried) - : query && retryRoutines.has(error.routine) + : query && query.prepared && retryRoutines.has(error.routine) ? retry(query, error) : errored(error) } diff --git a/src/index.js b/src/index.js index 0573e2bc..944d50cf 100644 --- a/src/index.js +++ b/src/index.js @@ -204,9 +204,10 @@ function Postgres(a, b) { const queue = Queue() const c = open.length ? open.shift() - : await new Promise(r => { - queries.push({ reserve: r }) - closed.length && connect(closed.shift()) + : await new Promise((resolve, reject) => { + const query = { reserve: resolve, reject } + queries.push(query) + closed.length && connect(closed.shift(), query) }) move(c, reserved) @@ -480,7 +481,7 @@ function parseOptions(a, b) { {} ), connection : { - application_name: 'postgres.js', + application_name: env.PGAPPNAME || 'postgres.js', ...o.connection, ...Object.entries(query).reduce((acc, [k, v]) => (k in defaults || (acc[k] = v), acc), {}) }, diff --git a/src/subscribe.js b/src/subscribe.js index 7a70842e..4f8934cc 100644 --- a/src/subscribe.js +++ b/src/subscribe.js @@ -47,7 +47,7 @@ export default function Subscribe(postgres, options) { return subscribe - async function subscribe(event, fn, onsubscribe = noop) { + async function subscribe(event, fn, onsubscribe = noop, onerror = noop) { event = parseEvent(event) if (!connection) @@ -66,6 +66,7 @@ export default function Subscribe(postgres, options) { return connection.then(x => { connected(x) onsubscribe() + stream && stream.on('error', onerror) return { unsubscribe, state, sql } }) } @@ -103,14 +104,16 @@ export default function Subscribe(postgres, options) { return { stream, state: xs.state } function error(e) { - console.error('Unexpected error during logical streaming - reconnecting', e) + console.error('Unexpected error during logical streaming - reconnecting', e) // eslint-disable-line } function data(x) { - if (x[0] === 0x77) + if (x[0] === 0x77) { parse(x.subarray(25), state, sql.options.parsers, handle, options.transform) - else if (x[0] === 0x6b && x[17]) + } else if (x[0] === 0x6b && x[17]) { + state.lsn = x.subarray(1, 9) pong() + } } function handle(a, b) { diff --git a/tests/index.js b/tests/index.js index e47cb534..07ff98ed 100644 --- a/tests/index.js +++ b/tests/index.js @@ -429,6 +429,30 @@ t('Reconnect using SSL', { timeout: 2 }, async() => { return [1, (await sql`select 1 as x`)[0].x] }) +t('Proper handling of non object Errors', async() => { + const sql = postgres({ socket: () => { throw 'wat' } }) // eslint-disable-line + + return [ + 'wat', await sql`select 1 as x`.catch(e => e.message) + ] +}) + +t('Proper handling of null Errors', async() => { + const sql = postgres({ socket: () => { throw null } }) // eslint-disable-line + + return [ + 'null', await sql`select 1 as x`.catch(e => e.message) + ] +}) + +t('Ensure reserve on connection throws proper error', async() => { + const sql = postgres({ socket: () => { throw 'wat' }, idle_timeout }) // eslint-disable-line + + return [ + 'wat', await sql.reserve().catch(e => e) + ] +}) + t('Login without password', async() => { return [true, (await postgres({ ...options, ...login })`select true as x`)[0].x] }) @@ -1789,6 +1813,32 @@ t('Recreate prepared statements on RevalidateCachedQuery error', async() => { ] }) +t('Properly throws routine error on not prepared statements', async() => { + await sql`create table x (x text[])` + const { routine } = await sql.unsafe(` + insert into x(x) values (('a', 'b')) + `).catch(e => e) + + return ['transformAssignedExpr', routine, await sql`drop table x`] +}) + +t('Properly throws routine error on not prepared statements in transaction', async() => { + const { routine } = await sql.begin(sql => [ + sql`create table x (x text[])`, + sql`insert into x(x) values (('a', 'b'))` + ]).catch(e => e) + + return ['transformAssignedExpr', routine] +}) + +t('Properly throws routine error on not prepared statements using file', async() => { + const { routine } = await sql.unsafe(` + create table x (x text[]); + insert into x(x) values (('a', 'b')); + `, { prepare: true }).catch(e => e) + + return ['transformAssignedExpr', routine] +}) t('Catches connection config errors', async() => { const sql = postgres({ ...options, user: { toString: () => { throw new Error('wat') } }, database: 'prut' }) @@ -2543,3 +2593,24 @@ t('reserve connection', async() => { xs.map(x => x.x).join('') ] }) + +t('arrays in reserved connection', async() => { + const reserved = await sql.reserve() + const [{ x }] = await reserved`select array[1, 2, 3] as x` + reserved.release() + + return [ + '123', + x.join('') + ] +}) + +t('Ensure reserve on query throws proper error', async() => { + const sql = postgres({ idle_timeout }) // eslint-disable-line + const reserved = await sql.reserve() + const [{ x }] = await reserved`select 'wat' as x` + + return [ + 'wat', x, reserved.release() + ] +}) diff --git a/transpile.deno.js b/transpile.deno.js index 923ac9af..f077677b 100644 --- a/transpile.deno.js +++ b/transpile.deno.js @@ -55,7 +55,7 @@ function transpile(x, name, folder) { .replace('{ spawnSync }', '{ spawn }') } if (name === 'index.js') - x += '\n;window.addEventListener("unload", () => Deno.exit(process.exitCode))' + x += '\n;globalThis.addEventListener("unload", () => Deno.exit(process.exitCode))' } const buffer = x.includes('Buffer') diff --git a/types/index.d.ts b/types/index.d.ts index 78d559ef..eb604918 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -456,7 +456,8 @@ declare namespace postgres { | 'NOT_TAGGED_CALL' | 'UNDEFINED_VALUE' | 'MAX_PARAMETERS_EXCEEDED' - | 'SASL_SIGNATURE_MISMATCH'; + | 'SASL_SIGNATURE_MISMATCH' + | 'UNSAFE_TRANSACTION'; message: string; } @@ -599,6 +600,7 @@ declare namespace postgres { type RowList = T & Iterable> & ResultQueryMeta; interface PendingQueryModifiers { + simple(): this; readable(): Promise; writable(): Promise; @@ -690,7 +692,7 @@ declare namespace postgres { listen(channel: string, onnotify: (value: string) => void, onlisten?: (() => void) | undefined): ListenRequest; notify(channel: string, payload: string): PendingRequest; - subscribe(event: string, cb: (row: Row | null, info: ReplicationEvent) => void, onsubscribe?: (() => void) | undefined): Promise; + subscribe(event: string, cb: (row: Row | null, info: ReplicationEvent) => void, onsubscribe?: (() => void), onerror?: (() => any)): Promise; largeObject(oid?: number | undefined, /** @default 0x00020000 | 0x00040000 */ mode?: number | undefined): Promise;