From b4c2526ba13e4f20f286b087887510e36c855f2f Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Mon, 4 Sep 2023 10:57:59 +0200 Subject: [PATCH 01/88] Improve notice search --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b0e64a75..32d3949c 100644 --- a/README.md +++ b/README.md @@ -941,7 +941,7 @@ const sql = postgres('postgres://username:password@host:port/database', { connect_timeout : 30, // Connect timeout in seconds prepare : true, // Automatic creation of prepared statements types : [], // Array of custom types, see more below - onnotice : fn, // Defaults to console.log + onnotice : fn, // Default console.log, set false to silence NOTICE onparameter : fn, // (key, value) when server param change debug : fn, // Is called with (connection, query, params, types) socket : fn, // fn returning custom socket to use From bf082a5c0ffe214924cd54752a7aeb4e618d279b Mon Sep 17 00:00:00 2001 From: Jorrit Posthuma Date: Tue, 12 Sep 2023 09:59:20 +0200 Subject: [PATCH 02/88] Fix connection on deno 1.36.3 (#673) --- src/connection.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connection.js b/src/connection.js index b4d0f6f1..e8e4881d 100644 --- a/src/connection.js +++ b/src/connection.js @@ -129,7 +129,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose try { x = options.socket ? (await Promise.resolve(options.socket(options))) - : net.Socket() + : new net.Socket() } catch (e) { error(e) return From 26c368e5a4ae533041232d30d43cfda838564ef1 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 12 Sep 2023 20:00:42 +1200 Subject: [PATCH 03/88] add docs and types for .reserve() (#667) --- README.md | 17 +++++++++++++++++ types/index.d.ts | 6 ++++++ 2 files changed, 23 insertions(+) diff --git a/README.md b/README.md index 32d3949c..da8df952 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ async function insertUser({ name, age }) { * [Teardown / Cleanup](#teardown--cleanup) * [Error handling](#error-handling) * [TypeScript support](#typescript-support) +* [Reserving connections](#reserving-connections) * [Changelog](./CHANGELOG.md) @@ -1151,6 +1152,22 @@ prexit(async () => { }) ``` +## Reserving connections + +### `await sql.reserve()` + +The `reserve` method pulls out a connection from the pool, and returns a client that wraps the single connection. This can be used for running queries on an isolated connection. + +```ts +const reserved = await sql.reserve() +await reserved`select * from users` +await reserved.release() +``` + +### `reserved.release()` + +Once you have finished with the reserved connection, call `release` to add it back to the pool. + ## Error handling Errors are all thrown to related queries and never globally. Errors coming from database itself are always in the [native Postgres format](https://www.postgresql.org/docs/current/errcodes-appendix.html), and the same goes for any [Node.js errors](https://nodejs.org/api/errors.html#errors_common_system_errors) eg. coming from the underlying connection. diff --git a/types/index.d.ts b/types/index.d.ts index ab797ee4..d76cb3b2 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -683,6 +683,8 @@ declare namespace postgres { file(path: string | Buffer | URL | number, options?: { cache?: boolean | undefined } | undefined): PendingQuery; file(path: string | Buffer | URL | number, args: (ParameterOrJSON)[], options?: { cache?: boolean | undefined } | undefined): PendingQuery; json(value: JSONValue): Parameter; + + reserve(): Promise> } interface UnsafeQueryOptions { @@ -699,6 +701,10 @@ declare namespace postgres { prepare(name: string): Promise>; } + + interface ReservedSql = {}> extends Sql { + release(): void; + } } export = postgres; From a3b30317e1ec968e0160a19f5aff2197000b4b19 Mon Sep 17 00:00:00 2001 From: MarisaCodes <103976925+MarisaCodes@users.noreply.github.com> Date: Tue, 12 Sep 2023 12:01:34 +0400 Subject: [PATCH 04/88] Update README.md (Transactions added missing returning *) (#662) Check issue: https://github.com/porsager/postgres/issues/649 This is a minor modification but debugging this has taken a couple of hours for me as I am slightly new to SQL syntax and to postgreSQL in general. I was trying to use the empty return from sql.begin but it turned out that the callback in sql.begin was the one returning the empty array even though the insert was successful. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index da8df952..6f5748b4 100644 --- a/README.md +++ b/README.md @@ -580,6 +580,7 @@ const [user, account] = await sql.begin(async sql => { ) values ( 'Murray' ) + returning * ` const [account] = await sql` @@ -588,6 +589,7 @@ const [user, account] = await sql.begin(async sql => { ) values ( ${ user.user_id } ) + returning * ` return [user, account] From 544f58b99739e4c356a50c9aa8d974f56a761c83 Mon Sep 17 00:00:00 2001 From: Miguel Victor Date: Tue, 12 Sep 2023 10:02:11 +0200 Subject: [PATCH 05/88] Fixed typo in README.md (#651) * Fixed typo in README.md * Updated sentence --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6f5748b4..45edb10e 100644 --- a/README.md +++ b/README.md @@ -235,7 +235,7 @@ update users set "name" = $1, "age" = $2 where user_id = $3 ``` ### Multiple updates in one query -It's possible to create multiple udpates in a single query. It's necessary to use arrays intead of objects to ensure the order of the items so that these correspond with the column names. +To create multiple updates in a single query, it is necessary to use arrays instead of objects to ensure that the order of the items correspond with the column names. ```js const users = [ [1, 'John', 34], From 4265251ca63ce76d2fb02e61ad0eeb686a116872 Mon Sep 17 00:00:00 2001 From: Paulo Vieira Date: Tue, 12 Sep 2023 09:02:41 +0100 Subject: [PATCH 06/88] Update README.md (prepared transactions) (#637) * Update README.md (prepared statements) - correct typo - add link to the official docs - change the subsection name to "PREPARE TRANSACTION" instead of "PREPARE" (because "PREPARE" is more associated with "prepared statements") One thing that is still a bit confusing in this section is the final sentence "Do note that you can often achieve...". It seems like it is referring to the "PREPARE" subsection, but in reality it is referring to the initial "BEGIN / COMMIT" subsection, no? * Update README.md - moved "Do note that you can often achieve the same result" to the first subsection - added a link to a question in dba.stackexchange.com that shows how to do it - in the insert and update example with dynamic columns, clarify that the columns can be given as an array * Update README.md - remove example in stackoverflow --- README.md | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 45edb10e..af97f69b 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ const user = { age: 68 } -sql` +await sql` insert into users ${ sql(user, 'name', 'age') } @@ -184,6 +184,15 @@ sql` // Which results in: insert into users ("name", "age") values ($1, $2) + +// The columns can also be given with an array +const columns = ['name', 'age'] + +await sql` + insert into users ${ + sql(user, columns) + } +` ``` **You can omit column names and simply execute `sql(user)` to get all the fields from the object as columns**. Be careful not to allow users to supply columns that you do not want to be inserted. @@ -223,7 +232,7 @@ const user = { age: 68 } -sql` +await sql` update users set ${ sql(user, 'name', 'age') } @@ -232,6 +241,16 @@ sql` // Which results in: update users set "name" = $1, "age" = $2 where user_id = $3 + +// The columns can also be given with an array +const columns = ['name', 'age'] + +await sql` + update users set ${ + sql(user, columns) + } + where user_id = ${ user.id } +` ``` ### Multiple updates in one query @@ -596,6 +615,8 @@ const [user, account] = await sql.begin(async sql => { }) ``` +Do note that you can often achieve the same result using [`WITH` queries (Common Table Expressions)](https://www.postgresql.org/docs/current/queries-with.html) instead of using transactions. + It's also possible to pipeline the requests in a transaction if needed by returning an array with queries from the callback function like this: ```js @@ -641,9 +662,9 @@ sql.begin('read write', async sql => { ``` -#### PREPARE `await sql.prepare([name]) -> fn()` +#### PREPARE TRANSACTION `await sql.prepare([name]) -> fn()` -Indicates that the transactions should be prepared using the `PREPARED TRANASCTION [NAME]` statement +Indicates that the transactions should be prepared using the [`PREPARE TRANSACTION [NAME]`](https://www.postgresql.org/docs/current/sql-prepare-transaction.html) statement instead of being committed. ```js @@ -660,8 +681,6 @@ sql.begin('read write', async sql => { }) ``` -Do note that you can often achieve the same result using [`WITH` queries (Common Table Expressions)](https://www.postgresql.org/docs/current/queries-with.html) instead of using transactions. - ## Data Transformation Postgres.js allows for transformation of the data passed to or returned from a query by using the `transform` option. From d26f8b4142d21105a0f0be8e7e4e4be074d43aa4 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 12 Sep 2023 10:09:30 +0200 Subject: [PATCH 07/88] Expose Socket from deno polyfill as class --- deno/polyfills.js | 262 ++++++++++++++++++++++++---------------------- 1 file changed, 135 insertions(+), 127 deletions(-) diff --git a/deno/polyfills.js b/deno/polyfills.js index 81da6c4c..71ee694d 100644 --- a/deno/polyfills.js +++ b/deno/polyfills.js @@ -5,6 +5,140 @@ import { isIP } from 'https://deno.land/std@0.132.0/node/net.ts' const events = () => ({ data: [], error: [], drain: [], connect: [], secureConnect: [], close: [] }) +class Socket { + constructor() { + return createSocket() + } +} + +function createSocket() { + let paused + , resume + , keepAlive + + const socket = { + error, + success, + readyState: 'open', + setKeepAlive: x => { + keepAlive = x + socket.raw && socket.raw.setKeepAlive && socket.raw.setKeepAlive(x) + }, + connect: (port, hostname) => { + socket.raw = null + socket.readyState = 'connecting' + typeof port === 'string' + ? Deno.connect({ transport: 'unix', path: socket.path = port }).then(success, error) + : Deno.connect({ transport: 'tcp', port: socket.port = port, hostname: socket.hostname = hostname || 'localhost' }).then(success, error) // eslint-disable-line + return socket + }, + pause: () => { + paused = new Promise(r => resume = r) + }, + resume: () => { + resume && resume() + paused = null + }, + isPaused: () => !!paused, + removeAllListeners: () => socket.events = events(), + events: events(), + raw: null, + on: (x, fn) => socket.events[x].push(fn), + once: (x, fn) => { + if (x === 'data') + socket.break = true + const e = socket.events[x] + e.push(once) + once.once = fn + function once(...args) { + fn(...args) + e.indexOf(once) > -1 && e.splice(e.indexOf(once), 1) + } + }, + removeListener: (x, fn) => { + socket.events[x] = socket.events[x].filter(x => x !== fn && x.once !== fn) + }, + write: (x, cb) => { + socket.raw.write(x).then(l => { + l < x.length + ? socket.write(x.slice(l), cb) + : (cb && cb(null)) + }).catch(err => { + cb && cb() + call(socket.events.error, err) + }) + return false + }, + destroy: () => close(), + end: (x) => { + x && socket.write(x) + close() + } + } + + return socket + + async function success(raw) { + if (socket.readyState !== 'connecting') + return raw.close() + + const encrypted = socket.encrypted + socket.raw = raw + keepAlive != null && raw.setKeepAlive && raw.setKeepAlive(keepAlive) + socket.readyState = 'open' + socket.encrypted + ? call(socket.events.secureConnect) + : call(socket.events.connect) + + const b = new Uint8Array(1024) + let result + + try { + while ((result = socket.readyState === 'open' && await raw.read(b))) { + call(socket.events.data, Buffer.from(b.subarray(0, result))) + if (!encrypted && socket.break && (socket.break = false, b[0] === 83)) + return socket.break = false + paused && await paused + } + } catch (e) { + if (e instanceof Deno.errors.BadResource === false) + error(e) + } + + if (!socket.encrypted || encrypted) + closed() + } + + function close() { + try { + socket.raw && socket.raw.close() + } catch (e) { + if (e instanceof Deno.errors.BadResource === false) + call(socket.events.error, e) + } + } + + function closed() { + if (socket.readyState === 'closed') + return + + socket.break = socket.encrypted = false + socket.readyState = 'closed' + call(socket.events.close) + } + + function error(err) { + call(socket.events.error, err) + socket.raw + ? close() + : closed() + } + + function call(xs, x) { + xs.slice().forEach(fn => fn(x)) + } +} + export const net = { isIP, createServer() { @@ -23,133 +157,7 @@ export const net = { } return server }, - Socket() { - let paused - , resume - , keepAlive - - const socket = { - error, - success, - readyState: 'open', - setKeepAlive: x => { - keepAlive = x - socket.raw && socket.raw.setKeepAlive && socket.raw.setKeepAlive(x) - }, - connect: (port, hostname) => { - socket.raw = null - socket.readyState = 'connecting' - typeof port === 'string' - ? Deno.connect({ transport: 'unix', path: socket.path = port }).then(success, error) - : Deno.connect({ transport: 'tcp', port: socket.port = port, hostname: socket.hostname = hostname || 'localhost' }).then(success, error) // eslint-disable-line - return socket - }, - pause: () => { - paused = new Promise(r => resume = r) - }, - resume: () => { - resume && resume() - paused = null - }, - isPaused: () => !!paused, - removeAllListeners: () => socket.events = events(), - events: events(), - raw: null, - on: (x, fn) => socket.events[x].push(fn), - once: (x, fn) => { - if (x === 'data') - socket.break = true - const e = socket.events[x] - e.push(once) - once.once = fn - function once(...args) { - fn(...args) - e.indexOf(once) > -1 && e.splice(e.indexOf(once), 1) - } - }, - removeListener: (x, fn) => { - socket.events[x] = socket.events[x].filter(x => x !== fn && x.once !== fn) - }, - write: (x, cb) => { - socket.raw.write(x).then(l => { - l < x.length - ? socket.write(x.slice(l), cb) - : (cb && cb(null)) - }).catch(err => { - cb && cb() - call(socket.events.error, err) - }) - return false - }, - destroy: () => close(), - end: (x) => { - x && socket.write(x) - close() - } - } - - return socket - - async function success(raw) { - if (socket.readyState !== 'connecting') - return raw.close() - - const encrypted = socket.encrypted - socket.raw = raw - keepAlive != null && raw.setKeepAlive && raw.setKeepAlive(keepAlive) - socket.readyState = 'open' - socket.encrypted - ? call(socket.events.secureConnect) - : call(socket.events.connect) - - const b = new Uint8Array(1024) - let result - - try { - while ((result = socket.readyState === 'open' && await raw.read(b))) { - call(socket.events.data, Buffer.from(b.subarray(0, result))) - if (!encrypted && socket.break && (socket.break = false, b[0] === 83)) - return socket.break = false - paused && await paused - } - } catch (e) { - if (e instanceof Deno.errors.BadResource === false) - error(e) - } - - if (!socket.encrypted || encrypted) - closed() - } - - function close() { - try { - socket.raw && socket.raw.close() - } catch (e) { - if (e instanceof Deno.errors.BadResource === false) - call(socket.events.error, e) - } - } - - function closed() { - if (socket.readyState === 'closed') - return - - socket.break = socket.encrypted = false - socket.readyState = 'closed' - call(socket.events.close) - } - - function error(err) { - call(socket.events.error, err) - socket.raw - ? close() - : closed() - } - - function call(xs, x) { - xs.slice().forEach(fn => fn(x)) - } - } + Socket } export const tls = { From 8b8a133aa46d2fad4f8bbf3584bb83ae8667d129 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 12 Sep 2023 10:16:56 +0200 Subject: [PATCH 08/88] Use new with net.Socket --- tests/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/index.js b/tests/index.js index d1d72b53..499b3fbd 100644 --- a/tests/index.js +++ b/tests/index.js @@ -2352,7 +2352,7 @@ t('Custom socket', {}, async() => { let result const sql = postgres({ socket: () => new Promise((resolve, reject) => { - const socket = net.Socket() + const socket = new net.Socket() socket.connect(5432) socket.once('data', x => result = x[0]) socket.on('error', reject) From 519575a58439b05cd82292da38dd506c1a890b88 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 12 Sep 2023 10:17:53 +0200 Subject: [PATCH 09/88] build --- cf/src/connection.js | 2 +- cjs/src/connection.js | 2 +- cjs/tests/index.js | 2 +- deno/README.md | 54 +++++++++++++++++++++++++++++++++++------- deno/src/connection.js | 2 +- deno/tests/index.js | 2 +- deno/types/index.d.ts | 6 +++++ 7 files changed, 57 insertions(+), 13 deletions(-) diff --git a/cf/src/connection.js b/cf/src/connection.js index 3803c8eb..c09b2720 100644 --- a/cf/src/connection.js +++ b/cf/src/connection.js @@ -131,7 +131,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose try { x = options.socket ? (await Promise.resolve(options.socket(options))) - : net.Socket() + : new net.Socket() } catch (e) { error(e) return diff --git a/cjs/src/connection.js b/cjs/src/connection.js index fc97a19b..5e3f26d0 100644 --- a/cjs/src/connection.js +++ b/cjs/src/connection.js @@ -129,7 +129,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose try { x = options.socket ? (await Promise.resolve(options.socket(options))) - : net.Socket() + : new net.Socket() } catch (e) { error(e) return diff --git a/cjs/tests/index.js b/cjs/tests/index.js index a8828d55..cb91c5c5 100644 --- a/cjs/tests/index.js +++ b/cjs/tests/index.js @@ -2352,7 +2352,7 @@ t('Custom socket', {}, async() => { let result const sql = postgres({ socket: () => new Promise((resolve, reject) => { - const socket = net.Socket() + const socket = new net.Socket() socket.connect(5432) socket.once('data', x => result = x[0]) socket.on('error', reject) diff --git a/deno/README.md b/deno/README.md index f599a18f..4c6d0fc8 100644 --- a/deno/README.md +++ b/deno/README.md @@ -75,6 +75,7 @@ async function insertUser({ name, age }) { * [Teardown / Cleanup](#teardown--cleanup) * [Error handling](#error-handling) * [TypeScript support](#typescript-support) +* [Reserving connections](#reserving-connections) * [Changelog](./CHANGELOG.md) @@ -171,7 +172,7 @@ const user = { age: 68 } -sql` +await sql` insert into users ${ sql(user, 'name', 'age') } @@ -179,6 +180,15 @@ sql` // Which results in: insert into users ("name", "age") values ($1, $2) + +// The columns can also be given with an array +const columns = ['name', 'age'] + +await sql` + insert into users ${ + sql(user, columns) + } +` ``` **You can omit column names and simply execute `sql(user)` to get all the fields from the object as columns**. Be careful not to allow users to supply columns that you do not want to be inserted. @@ -218,7 +228,7 @@ const user = { age: 68 } -sql` +await sql` update users set ${ sql(user, 'name', 'age') } @@ -227,10 +237,20 @@ sql` // Which results in: update users set "name" = $1, "age" = $2 where user_id = $3 + +// The columns can also be given with an array +const columns = ['name', 'age'] + +await sql` + update users set ${ + sql(user, columns) + } + where user_id = ${ user.id } +` ``` ### Multiple updates in one query -It's possible to create multiple udpates in a single query. It's necessary to use arrays intead of objects to ensure the order of the items so that these correspond with the column names. +To create multiple updates in a single query, it is necessary to use arrays instead of objects to ensure that the order of the items correspond with the column names. ```js const users = [ [1, 'John', 34], @@ -575,6 +595,7 @@ const [user, account] = await sql.begin(async sql => { ) values ( 'Murray' ) + returning * ` const [account] = await sql` @@ -583,12 +604,15 @@ const [user, account] = await sql.begin(async sql => { ) values ( ${ user.user_id } ) + returning * ` return [user, account] }) ``` +Do note that you can often achieve the same result using [`WITH` queries (Common Table Expressions)](https://www.postgresql.org/docs/current/queries-with.html) instead of using transactions. + It's also possible to pipeline the requests in a transaction if needed by returning an array with queries from the callback function like this: ```js @@ -634,9 +658,9 @@ sql.begin('read write', async sql => { ``` -#### PREPARE `await sql.prepare([name]) -> fn()` +#### PREPARE TRANSACTION `await sql.prepare([name]) -> fn()` -Indicates that the transactions should be prepared using the `PREPARED TRANASCTION [NAME]` statement +Indicates that the transactions should be prepared using the [`PREPARE TRANSACTION [NAME]`](https://www.postgresql.org/docs/current/sql-prepare-transaction.html) statement instead of being committed. ```js @@ -653,8 +677,6 @@ sql.begin('read write', async sql => { }) ``` -Do note that you can often achieve the same result using [`WITH` queries (Common Table Expressions)](https://www.postgresql.org/docs/current/queries-with.html) instead of using transactions. - ## Data Transformation Postgres.js allows for transformation of the data passed to or returned from a query by using the `transform` option. @@ -937,7 +959,7 @@ const sql = postgres('postgres://username:password@host:port/database', { connect_timeout : 30, // Connect timeout in seconds prepare : true, // Automatic creation of prepared statements types : [], // Array of custom types, see more below - onnotice : fn, // Defaults to console.log + onnotice : fn, // Default console.log, set false to silence NOTICE onparameter : fn, // (key, value) when server param change debug : fn, // Is called with (connection, query, params, types) socket : fn, // fn returning custom socket to use @@ -1147,6 +1169,22 @@ prexit(async () => { }) ``` +## Reserving connections + +### `await sql.reserve()` + +The `reserve` method pulls out a connection from the pool, and returns a client that wraps the single connection. This can be used for running queries on an isolated connection. + +```ts +const reserved = await sql.reserve() +await reserved`select * from users` +await reserved.release() +``` + +### `reserved.release()` + +Once you have finished with the reserved connection, call `release` to add it back to the pool. + ## Error handling Errors are all thrown to related queries and never globally. Errors coming from database itself are always in the [native Postgres format](https://www.postgresql.org/docs/current/errcodes-appendix.html), and the same goes for any [Node.js errors](https://nodejs.org/api/errors.html#errors_common_system_errors) eg. coming from the underlying connection. diff --git a/deno/src/connection.js b/deno/src/connection.js index 80382577..95e73dda 100644 --- a/deno/src/connection.js +++ b/deno/src/connection.js @@ -132,7 +132,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose try { x = options.socket ? (await Promise.resolve(options.socket(options))) - : net.Socket() + : new net.Socket() } catch (e) { error(e) return diff --git a/deno/tests/index.js b/deno/tests/index.js index 210a9f9b..08a0c023 100644 --- a/deno/tests/index.js +++ b/deno/tests/index.js @@ -2354,7 +2354,7 @@ t('Custom socket', {}, async() => { let result const sql = postgres({ socket: () => new Promise((resolve, reject) => { - const socket = net.Socket() + const socket = new net.Socket() socket.connect(5432) socket.once('data', x => result = x[0]) socket.on('error', reject) diff --git a/deno/types/index.d.ts b/deno/types/index.d.ts index 64a00a4c..0fb74e03 100644 --- a/deno/types/index.d.ts +++ b/deno/types/index.d.ts @@ -685,6 +685,8 @@ declare namespace postgres { file(path: string | Buffer | URL | number, options?: { cache?: boolean | undefined } | undefined): PendingQuery; file(path: string | Buffer | URL | number, args: (ParameterOrJSON)[], options?: { cache?: boolean | undefined } | undefined): PendingQuery; json(value: JSONValue): Parameter; + + reserve(): Promise> } interface UnsafeQueryOptions { @@ -701,6 +703,10 @@ declare namespace postgres { prepare(name: string): Promise>; } + + interface ReservedSql = {}> extends Sql { + release(): void; + } } export = postgres; From 989ec55b80cf4f21465132289191891d95c8d790 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 19 Sep 2023 13:57:41 +0200 Subject: [PATCH 10/88] Try Postgres 16 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 92ec7033..85a859ff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ jobs: fail-fast: false matrix: node: ['12', '14', '16', '18', '20'] - postgres: ['12', '13', '14', '15'] + postgres: ['12', '13', '14', '15', '16'] runs-on: ubuntu-latest services: postgres: From e4b158be1fb2333d99a6a30e33b6acf3476b3dbb Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 10 Oct 2023 15:02:21 +0200 Subject: [PATCH 11/88] Allow a falsy url string --- src/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/index.js b/src/index.js index 936be5cc..54513e10 100644 --- a/src/index.js +++ b/src/index.js @@ -427,7 +427,7 @@ function parseOptions(a, b) { return a const env = process.env // eslint-disable-line - , o = (typeof a === 'string' ? b : a) || {} + , o = (!a || typeof a === 'string' ? b : a) || {} , { url, multihost } = parseUrl(a) , query = [...url.searchParams].reduce((a, [b, c]) => (a[b] = c, a), {}) , host = o.hostname || o.host || multihost || url.hostname || env.PGHOST || 'localhost' @@ -528,7 +528,7 @@ function parseTransform(x) { } function parseUrl(url) { - if (typeof url !== 'string') + if (!url || typeof url !== 'string') return { url: { searchParams: new Map() } } let host = url From ded413f1e235b519b7ae40602f346216a97fce8d Mon Sep 17 00:00:00 2001 From: Alessandro Cosentino Date: Tue, 10 Oct 2023 15:12:33 +0200 Subject: [PATCH 12/88] Fix reserved connection query handler (#683) --- cf/src/index.js | 8 ++++---- cjs/src/index.js | 8 ++++---- deno/src/index.js | 8 ++++---- src/index.js | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cf/src/index.js b/cf/src/index.js index da4df290..e35c899d 100644 --- a/cf/src/index.js +++ b/cf/src/index.js @@ -202,7 +202,7 @@ function Postgres(a, b) { } async function reserve() { - const q = Queue() + const queue = Queue() const c = open.length ? open.shift() : await new Promise(r => { @@ -211,8 +211,8 @@ function Postgres(a, b) { }) move(c, reserved) - c.reserved = () => q.length - ? c.execute(q.shift()) + c.reserved = () => queue.length + ? c.execute(queue.shift()) : move(c, reserved) c.reserved.release = true @@ -226,7 +226,7 @@ function Postgres(a, b) { function handler(q) { c.queue === full - ? q.push(q) + ? queue.push(q) : c.execute(q) || move(c, full) } } diff --git a/cjs/src/index.js b/cjs/src/index.js index d022b976..17595880 100644 --- a/cjs/src/index.js +++ b/cjs/src/index.js @@ -201,7 +201,7 @@ function Postgres(a, b) { } async function reserve() { - const q = Queue() + const queue = Queue() const c = open.length ? open.shift() : await new Promise(r => { @@ -210,8 +210,8 @@ function Postgres(a, b) { }) move(c, reserved) - c.reserved = () => q.length - ? c.execute(q.shift()) + c.reserved = () => queue.length + ? c.execute(queue.shift()) : move(c, reserved) c.reserved.release = true @@ -225,7 +225,7 @@ function Postgres(a, b) { function handler(q) { c.queue === full - ? q.push(q) + ? queue.push(q) : c.execute(q) || move(c, full) } } diff --git a/deno/src/index.js b/deno/src/index.js index a871e0f1..9ad5a2f2 100644 --- a/deno/src/index.js +++ b/deno/src/index.js @@ -202,7 +202,7 @@ function Postgres(a, b) { } async function reserve() { - const q = Queue() + const queue = Queue() const c = open.length ? open.shift() : await new Promise(r => { @@ -211,8 +211,8 @@ function Postgres(a, b) { }) move(c, reserved) - c.reserved = () => q.length - ? c.execute(q.shift()) + c.reserved = () => queue.length + ? c.execute(queue.shift()) : move(c, reserved) c.reserved.release = true @@ -226,7 +226,7 @@ function Postgres(a, b) { function handler(q) { c.queue === full - ? q.push(q) + ? queue.push(q) : c.execute(q) || move(c, full) } } diff --git a/src/index.js b/src/index.js index 54513e10..ff990586 100644 --- a/src/index.js +++ b/src/index.js @@ -201,7 +201,7 @@ function Postgres(a, b) { } async function reserve() { - const q = Queue() + const queue = Queue() const c = open.length ? open.shift() : await new Promise(r => { @@ -210,8 +210,8 @@ function Postgres(a, b) { }) move(c, reserved) - c.reserved = () => q.length - ? c.execute(q.shift()) + c.reserved = () => queue.length + ? c.execute(queue.shift()) : move(c, reserved) c.reserved.release = true @@ -225,7 +225,7 @@ function Postgres(a, b) { function handler(q) { c.queue === full - ? q.push(q) + ? queue.push(q) : c.execute(q) || move(c, full) } } From 31f9856477a509122ca63216cd3aa2e158f8da21 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 10 Oct 2023 15:56:35 +0200 Subject: [PATCH 13/88] Clear roles on test bootstrap --- tests/bootstrap.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/bootstrap.js b/tests/bootstrap.js index 0070c7b7..f877543a 100644 --- a/tests/bootstrap.js +++ b/tests/bootstrap.js @@ -1,15 +1,19 @@ import { spawnSync } from 'child_process' +exec('dropdb', ['postgres_js_test']) + exec('psql', ['-c', 'alter system set ssl=on']) +exec('psql', ['-c', 'drop user postgres_js_test']) exec('psql', ['-c', 'create user postgres_js_test']) exec('psql', ['-c', 'alter system set password_encryption=md5']) exec('psql', ['-c', 'select pg_reload_conf()']) +exec('psql', ['-c', 'drop user if exists postgres_js_test_md5']) exec('psql', ['-c', 'create user postgres_js_test_md5 with password \'postgres_js_test_md5\'']) exec('psql', ['-c', 'alter system set password_encryption=\'scram-sha-256\'']) exec('psql', ['-c', 'select pg_reload_conf()']) +exec('psql', ['-c', 'drop user if exists postgres_js_test_scram']) exec('psql', ['-c', 'create user postgres_js_test_scram with password \'postgres_js_test_scram\'']) -exec('dropdb', ['postgres_js_test']) exec('createdb', ['postgres_js_test']) exec('psql', ['-c', 'grant all on database postgres_js_test to postgres_js_test']) exec('psql', ['-c', 'alter database postgres_js_test owner to postgres_js_test']) From 63ec056eb3655bed17511a4664bf8eb5e5be943b Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 10 Oct 2023 15:59:51 +0200 Subject: [PATCH 14/88] build --- cf/src/index.js | 4 ++-- cjs/src/index.js | 4 ++-- cjs/tests/bootstrap.js | 6 +++++- deno/src/index.js | 4 ++-- deno/tests/bootstrap.js | 6 +++++- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/cf/src/index.js b/cf/src/index.js index e35c899d..0c74f5cf 100644 --- a/cf/src/index.js +++ b/cf/src/index.js @@ -428,7 +428,7 @@ function parseOptions(a, b) { return a const env = process.env // eslint-disable-line - , o = (typeof a === 'string' ? b : a) || {} + , o = (!a || typeof a === 'string' ? b : a) || {} , { url, multihost } = parseUrl(a) , query = [...url.searchParams].reduce((a, [b, c]) => (a[b] = c, a), {}) , host = o.hostname || o.host || multihost || url.hostname || env.PGHOST || 'localhost' @@ -529,7 +529,7 @@ function parseTransform(x) { } function parseUrl(url) { - if (typeof url !== 'string') + if (!url || typeof url !== 'string') return { url: { searchParams: new Map() } } let host = url diff --git a/cjs/src/index.js b/cjs/src/index.js index 17595880..698b05d4 100644 --- a/cjs/src/index.js +++ b/cjs/src/index.js @@ -427,7 +427,7 @@ function parseOptions(a, b) { return a const env = process.env // eslint-disable-line - , o = (typeof a === 'string' ? b : a) || {} + , o = (!a || typeof a === 'string' ? b : a) || {} , { url, multihost } = parseUrl(a) , query = [...url.searchParams].reduce((a, [b, c]) => (a[b] = c, a), {}) , host = o.hostname || o.host || multihost || url.hostname || env.PGHOST || 'localhost' @@ -528,7 +528,7 @@ function parseTransform(x) { } function parseUrl(url) { - if (typeof url !== 'string') + if (!url || typeof url !== 'string') return { url: { searchParams: new Map() } } let host = url diff --git a/cjs/tests/bootstrap.js b/cjs/tests/bootstrap.js index 0ff56fbb..2106f0f8 100644 --- a/cjs/tests/bootstrap.js +++ b/cjs/tests/bootstrap.js @@ -1,15 +1,19 @@ const { spawnSync } = require('child_process') +exec('dropdb', ['postgres_js_test']) + exec('psql', ['-c', 'alter system set ssl=on']) +exec('psql', ['-c', 'drop user postgres_js_test']) exec('psql', ['-c', 'create user postgres_js_test']) exec('psql', ['-c', 'alter system set password_encryption=md5']) exec('psql', ['-c', 'select pg_reload_conf()']) +exec('psql', ['-c', 'drop user if exists postgres_js_test_md5']) exec('psql', ['-c', 'create user postgres_js_test_md5 with password \'postgres_js_test_md5\'']) exec('psql', ['-c', 'alter system set password_encryption=\'scram-sha-256\'']) exec('psql', ['-c', 'select pg_reload_conf()']) +exec('psql', ['-c', 'drop user if exists postgres_js_test_scram']) exec('psql', ['-c', 'create user postgres_js_test_scram with password \'postgres_js_test_scram\'']) -exec('dropdb', ['postgres_js_test']) exec('createdb', ['postgres_js_test']) exec('psql', ['-c', 'grant all on database postgres_js_test to postgres_js_test']) exec('psql', ['-c', 'alter database postgres_js_test owner to postgres_js_test']) diff --git a/deno/src/index.js b/deno/src/index.js index 9ad5a2f2..fada05ae 100644 --- a/deno/src/index.js +++ b/deno/src/index.js @@ -428,7 +428,7 @@ function parseOptions(a, b) { return a const env = process.env // eslint-disable-line - , o = (typeof a === 'string' ? b : a) || {} + , o = (!a || typeof a === 'string' ? b : a) || {} , { url, multihost } = parseUrl(a) , query = [...url.searchParams].reduce((a, [b, c]) => (a[b] = c, a), {}) , host = o.hostname || o.host || multihost || url.hostname || env.PGHOST || 'localhost' @@ -529,7 +529,7 @@ function parseTransform(x) { } function parseUrl(url) { - if (typeof url !== 'string') + if (!url || typeof url !== 'string') return { url: { searchParams: new Map() } } let host = url diff --git a/deno/tests/bootstrap.js b/deno/tests/bootstrap.js index 699b54bf..da416896 100644 --- a/deno/tests/bootstrap.js +++ b/deno/tests/bootstrap.js @@ -1,15 +1,19 @@ import { spawn } from 'https://deno.land/std@0.132.0/node/child_process.ts' +await exec('dropdb', ['postgres_js_test']) + await exec('psql', ['-c', 'alter system set ssl=on']) +await exec('psql', ['-c', 'drop user postgres_js_test']) await exec('psql', ['-c', 'create user postgres_js_test']) await exec('psql', ['-c', 'alter system set password_encryption=md5']) await exec('psql', ['-c', 'select pg_reload_conf()']) +await exec('psql', ['-c', 'drop user if exists postgres_js_test_md5']) await exec('psql', ['-c', 'create user postgres_js_test_md5 with password \'postgres_js_test_md5\'']) await exec('psql', ['-c', 'alter system set password_encryption=\'scram-sha-256\'']) await exec('psql', ['-c', 'select pg_reload_conf()']) +await exec('psql', ['-c', 'drop user if exists postgres_js_test_scram']) await exec('psql', ['-c', 'create user postgres_js_test_scram with password \'postgres_js_test_scram\'']) -await exec('dropdb', ['postgres_js_test']) await exec('createdb', ['postgres_js_test']) await exec('psql', ['-c', 'grant all on database postgres_js_test to postgres_js_test']) await exec('psql', ['-c', 'alter database postgres_js_test owner to postgres_js_test']) From 7bcb5b182d0f7da9363445d7fe88d879072ed2e1 Mon Sep 17 00:00:00 2001 From: Matt Silverlock Date: Tue, 10 Oct 2023 14:16:13 -0400 Subject: [PATCH 15/88] add Cloudflare Workers to README --- README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/README.md b/README.md index af97f69b..39678273 100644 --- a/README.md +++ b/README.md @@ -1060,6 +1060,34 @@ const sql = postgres({ }) ``` +### Cloudflare Workers support + +Postgres.js has built-in support for the [TCP socket API](https://developers.cloudflare.com/workers/runtime-apis/tcp-sockets/) in Cloudflare Workers, which is [on-track](https://github.com/wintercg/proposal-sockets-api) to be standardized and adopted in Node.js and other JavaScript runtimes, such as Deno. + +You can use Postgres.js directly in a Worker, or to benefit from connection pooling and query caching, via the [Hyperdrive](https://developers.cloudflare.com/hyperdrive/learning/connect-to-postgres/#driver-examples) service available to Workers by passing the Hyperdrive `connectionString` when creating a new `postgres` client as follows: + +```ts +// Requires Postgres.js 3.4.0 or later +import postgres from 'postgres' + +interface Env { + HYPERDRIVE: Hyperdrive; +} + +export default async fetch(req: Request, env: Env, ctx: ExecutionContext) { + // The Postgres.js library accepts a connection string directly + const sql = postgres(env.HYPERDRIVE.connectionString) + const results = await sql`SELECT * FROM users LIMIT 10` + return Response.json(results) +} +``` + +In `wrangler.toml` you will need to enable `node_compat` to allow Postgres.js to operate in the Workers environment: + +```toml +node_compat = true # required for database drivers to function +``` + ### Auto fetching of array types Postgres.js will automatically fetch table/array-type information when it first connects to a database. From 5f569d85bada8c84750f634f8ee3d47828fca17e Mon Sep 17 00:00:00 2001 From: Alexander Bolshakov Date: Mon, 18 Sep 2023 10:08:42 +0400 Subject: [PATCH 16/88] Fix #674 TypeScript issues with dynamic inserts --- types/index.d.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/types/index.d.ts b/types/index.d.ts index d76cb3b2..8dacd9c4 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -177,9 +177,17 @@ type Rest = T extends TemplateStringsArray ? never : // force fallback to the tagged template function overload T extends string ? readonly string[] : T extends readonly any[][] ? readonly [] : - T extends readonly (object & infer R)[] ? readonly (Keys & keyof R)[] : + T extends readonly (object & infer R)[] ? ( + readonly (Keys & keyof R)[] // sql(data, "prop", "prop2") syntax + | + [readonly (Keys & keyof R)[]] // sql(data, ["prop", "prop2"]) syntax + ) : T extends readonly any[] ? readonly [] : - T extends object ? readonly (Keys & keyof T)[] : + T extends object ? ( + readonly (Keys & keyof T)[] // sql(data, "prop", "prop2") syntax + | + [readonly (Keys & keyof T)[]] // sql(data, ["prop", "prop2"]) syntax + ) : any type Return = From cae4d9711d5109a794ca53b8ba4ec13305afdb10 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 10 Oct 2023 20:42:03 +0200 Subject: [PATCH 17/88] Fix a bun issue with stack traces --- src/query.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/query.js b/src/query.js index 848f3b88..0d44a15c 100644 --- a/src/query.js +++ b/src/query.js @@ -37,13 +37,12 @@ export class Query extends Promise { } get origin() { - return this.handler.debug + return (this.handler.debug ? this[originError].stack - : this.tagged - ? originStackCache.has(this.strings) - ? originStackCache.get(this.strings) - : originStackCache.set(this.strings, this[originError].stack).get(this.strings) - : '' + : this.tagged && originStackCache.has(this.strings) + ? originStackCache.get(this.strings) + : originStackCache.set(this.strings, this[originError].stack).get(this.strings) + ) || '' } static get [Symbol.species]() { From 92a8b6d844bdd201b1bb9cb688ff4bc7fa5192d2 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 10 Oct 2023 20:42:11 +0200 Subject: [PATCH 18/88] build build --- cf/src/query.js | 11 +++++------ cjs/src/query.js | 11 +++++------ deno/README.md | 28 ++++++++++++++++++++++++++++ deno/src/query.js | 11 +++++------ deno/types/index.d.ts | 12 ++++++++++-- 5 files changed, 53 insertions(+), 20 deletions(-) diff --git a/cf/src/query.js b/cf/src/query.js index 848f3b88..0d44a15c 100644 --- a/cf/src/query.js +++ b/cf/src/query.js @@ -37,13 +37,12 @@ export class Query extends Promise { } get origin() { - return this.handler.debug + return (this.handler.debug ? this[originError].stack - : this.tagged - ? originStackCache.has(this.strings) - ? originStackCache.get(this.strings) - : originStackCache.set(this.strings, this[originError].stack).get(this.strings) - : '' + : this.tagged && originStackCache.has(this.strings) + ? originStackCache.get(this.strings) + : originStackCache.set(this.strings, this[originError].stack).get(this.strings) + ) || '' } static get [Symbol.species]() { diff --git a/cjs/src/query.js b/cjs/src/query.js index 7246c5f3..45327f2f 100644 --- a/cjs/src/query.js +++ b/cjs/src/query.js @@ -37,13 +37,12 @@ const Query = module.exports.Query = class Query extends Promise { } get origin() { - return this.handler.debug + return (this.handler.debug ? this[originError].stack - : this.tagged - ? originStackCache.has(this.strings) - ? originStackCache.get(this.strings) - : originStackCache.set(this.strings, this[originError].stack).get(this.strings) - : '' + : this.tagged && originStackCache.has(this.strings) + ? originStackCache.get(this.strings) + : originStackCache.set(this.strings, this[originError].stack).get(this.strings) + ) || '' } static get [Symbol.species]() { diff --git a/deno/README.md b/deno/README.md index 4c6d0fc8..19fd0993 100644 --- a/deno/README.md +++ b/deno/README.md @@ -1056,6 +1056,34 @@ const sql = postgres({ }) ``` +### Cloudflare Workers support + +Postgres.js has built-in support for the [TCP socket API](https://developers.cloudflare.com/workers/runtime-apis/tcp-sockets/) in Cloudflare Workers, which is [on-track](https://github.com/wintercg/proposal-sockets-api) to be standardized and adopted in Node.js and other JavaScript runtimes, such as Deno. + +You can use Postgres.js directly in a Worker, or to benefit from connection pooling and query caching, via the [Hyperdrive](https://developers.cloudflare.com/hyperdrive/learning/connect-to-postgres/#driver-examples) service available to Workers by passing the Hyperdrive `connectionString` when creating a new `postgres` client as follows: + +```ts +// Requires Postgres.js 3.4.0 or later +import postgres from 'postgres' + +interface Env { + HYPERDRIVE: Hyperdrive; +} + +export default async fetch(req: Request, env: Env, ctx: ExecutionContext) { + // The Postgres.js library accepts a connection string directly + const sql = postgres(env.HYPERDRIVE.connectionString) + const results = await sql`SELECT * FROM users LIMIT 10` + return Response.json(results) +} +``` + +In `wrangler.toml` you will need to enable `node_compat` to allow Postgres.js to operate in the Workers environment: + +```toml +node_compat = true # required for database drivers to function +``` + ### Auto fetching of array types Postgres.js will automatically fetch table/array-type information when it first connects to a database. diff --git a/deno/src/query.js b/deno/src/query.js index 848f3b88..0d44a15c 100644 --- a/deno/src/query.js +++ b/deno/src/query.js @@ -37,13 +37,12 @@ export class Query extends Promise { } get origin() { - return this.handler.debug + return (this.handler.debug ? this[originError].stack - : this.tagged - ? originStackCache.has(this.strings) - ? originStackCache.get(this.strings) - : originStackCache.set(this.strings, this[originError].stack).get(this.strings) - : '' + : this.tagged && originStackCache.has(this.strings) + ? originStackCache.get(this.strings) + : originStackCache.set(this.strings, this[originError].stack).get(this.strings) + ) || '' } static get [Symbol.species]() { diff --git a/deno/types/index.d.ts b/deno/types/index.d.ts index 0fb74e03..215d5b62 100644 --- a/deno/types/index.d.ts +++ b/deno/types/index.d.ts @@ -179,9 +179,17 @@ type Rest = T extends TemplateStringsArray ? never : // force fallback to the tagged template function overload T extends string ? readonly string[] : T extends readonly any[][] ? readonly [] : - T extends readonly (object & infer R)[] ? readonly (Keys & keyof R)[] : + T extends readonly (object & infer R)[] ? ( + readonly (Keys & keyof R)[] // sql(data, "prop", "prop2") syntax + | + [readonly (Keys & keyof R)[]] // sql(data, ["prop", "prop2"]) syntax + ) : T extends readonly any[] ? readonly [] : - T extends object ? readonly (Keys & keyof T)[] : + T extends object ? ( + readonly (Keys & keyof T)[] // sql(data, "prop", "prop2") syntax + | + [readonly (Keys & keyof T)[]] // sql(data, ["prop", "prop2"]) syntax + ) : any type Return = From 0428b30937400a7dadc8ed09587c44ef917052a6 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 10 Oct 2023 20:44:41 +0200 Subject: [PATCH 19/88] 3.4.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c9d00db5..c7c8dcde 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "postgres", - "version": "3.3.5", + "version": "3.4.0", "description": "Fastest full featured PostgreSQL client for Node.js", "type": "module", "module": "src/index.js", From 09e6cb5247c514e5cf50faced6452fae956edeb9 Mon Sep 17 00:00:00 2001 From: Alex Robinson Date: Fri, 13 Oct 2023 16:39:00 -0500 Subject: [PATCH 20/88] Update Cloudflare createHash polyfill to support md5 and hex encoding Since the md5 method in cf/src/connection.js expects to be able to call crypto.createHash('md5').update(x).digest('hex') This was causing md5 password auth to hang when used from a Cloudflare worker, but now I've confirmed md5 password auth works. --- cf/polyfills.js | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/cf/polyfills.js b/cf/polyfills.js index f7809003..53c5203d 100644 --- a/cf/polyfills.js +++ b/cf/polyfills.js @@ -47,12 +47,25 @@ export const crypto = { ), createHash: type => ({ update: x => ({ - digest: () => { - if (type !== 'sha256') - throw Error('createHash only supports sha256 in this environment.') - if (!(x instanceof Uint8Array)) + digest: encoding => { + if (!(x instanceof Uint8Array)) { x = textEncoder.encode(x) - return Crypto.subtle.digest('SHA-256', x) + } + let prom + if (type === 'sha256') { + prom = Crypto.subtle.digest('SHA-256', x) + } else if (type === 'md5') { + prom = Crypto.subtle.digest('md5', x) + } else { + throw Error('createHash only supports sha256 or md5 in this environment, not ${type}.') + } + if (encoding === 'hex') { + return prom.then((arrayBuf) => Buffer.from(arrayBuf).toString('hex')) + } else if (encoding) { + throw Error(`createHash only supports hex encoding or unencoded in this environment, not ${encoding}`) + } else { + return prom + } } }) }), From c1d851901ed84f49f98328474fb324c3b10e476d Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Mon, 23 Oct 2023 23:24:47 +0200 Subject: [PATCH 21/88] Ensure bun imports esm instead of cf worker - fixes #692 --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index c7c8dcde..7989cd52 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "module": "src/index.js", "main": "cjs/src/index.js", "exports": { + "bun": "./src/index.js", "worker": "./cf/src/index.js", "types": "./types/index.d.ts", "import": "./src/index.js", From 00dd98a75e878c4421df3a72f0ad53ce95f060ca Mon Sep 17 00:00:00 2001 From: Luke Edwards Date: Mon, 23 Oct 2023 14:32:01 -0700 Subject: [PATCH 22/88] set "types" exports first as ts 4.7 requirement (#709) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7989cd52..11316987 100644 --- a/package.json +++ b/package.json @@ -6,9 +6,9 @@ "module": "src/index.js", "main": "cjs/src/index.js", "exports": { + "types": "./types/index.d.ts", "bun": "./src/index.js", "worker": "./cf/src/index.js", - "types": "./types/index.d.ts", "import": "./src/index.js", "default": "./cjs/src/index.js" }, From cb353f22e430cbbd56bbaa208cfc75b6e7534b3f Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Mon, 23 Oct 2023 23:36:40 +0200 Subject: [PATCH 23/88] Add engines.node --- package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package.json b/package.json index 11316987..28826d5c 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,9 @@ }, "types": "types/index.d.ts", "typings": "types/index.d.ts", + "engines": { + "node": ">=12" + }, "scripts": { "build": "npm run build:cjs && npm run build:deno && npm run build:cf", "build:cjs": "node transpile.cjs", From 428475aa0ced9234e8b7dd76daa3c91907ece08c Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Mon, 23 Oct 2023 23:37:26 +0200 Subject: [PATCH 24/88] 3.4.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 28826d5c..e8a552d1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "postgres", - "version": "3.4.0", + "version": "3.4.1", "description": "Fastest full featured PostgreSQL client for Node.js", "type": "module", "module": "src/index.js", From 33ae0ed204c2a7c5231dcc7e94af6d7ab3977eb2 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Fri, 27 Oct 2023 00:02:30 +0200 Subject: [PATCH 25/88] Fix race conditions when creating payloads - fixes #430 #668 --- src/connection.js | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/connection.js b/src/connection.js index e8e4881d..1135189f 100644 --- a/src/connection.js +++ b/src/connection.js @@ -656,27 +656,30 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose /* c8 ignore next 5 */ async function AuthenticationCleartextPassword() { + const payload = await Pass() write( - b().p().str(await Pass()).z(1).end() + b().p().str(payload).z(1).end() ) } async function AuthenticationMD5Password(x) { - write( - b().p().str( - 'md5' + - (await md5(Buffer.concat([ + const payload = 'md5' + ( + await md5( + Buffer.concat([ Buffer.from(await md5((await Pass()) + user)), x.subarray(9) - ]))) - ).z(1).end() + ]) + ) + ) + write( + b().p().str(payload).z(1).end() ) } async function SASL() { + nonce = (await crypto.randomBytes(18)).toString('base64') b().p().str('SCRAM-SHA-256' + b.N) const i = b.i - nonce = (await crypto.randomBytes(18)).toString('base64') write(b.inc(4).str('n,,n=*,r=' + nonce).i32(b.i - i - 4, i).end()) } @@ -698,12 +701,12 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose serverSignature = (await hmac(await hmac(saltedPassword, 'Server Key'), auth)).toString('base64') + const payload = 'c=biws,r=' + res.r + ',p=' + xor( + clientKey, Buffer.from(await hmac(await sha256(clientKey), auth)) + ).toString('base64') + write( - b().p().str( - 'c=biws,r=' + res.r + ',p=' + xor( - clientKey, Buffer.from(await hmac(await sha256(clientKey), auth)) - ).toString('base64') - ).end() + b().p().str(payload).end() ) } From 09441e743b66f6472cca92e0154eee3326ea0140 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Fri, 27 Oct 2023 00:14:23 +0200 Subject: [PATCH 26/88] build --- cjs/src/connection.js | 29 ++++++++++++++++------------- cjs/tests/index.js | 2 +- deno/src/connection.js | 29 ++++++++++++++++------------- deno/tests/index.js | 2 +- tests/index.js | 2 +- 5 files changed, 35 insertions(+), 29 deletions(-) diff --git a/cjs/src/connection.js b/cjs/src/connection.js index 5e3f26d0..c07d3027 100644 --- a/cjs/src/connection.js +++ b/cjs/src/connection.js @@ -656,27 +656,30 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose /* c8 ignore next 5 */ async function AuthenticationCleartextPassword() { + const payload = await Pass() write( - b().p().str(await Pass()).z(1).end() + b().p().str(payload).z(1).end() ) } async function AuthenticationMD5Password(x) { - write( - b().p().str( - 'md5' + - (await md5(Buffer.concat([ + const payload = 'md5' + ( + await md5( + Buffer.concat([ Buffer.from(await md5((await Pass()) + user)), x.subarray(9) - ]))) - ).z(1).end() + ]) + ) + ) + write( + b().p().str(payload).z(1).end() ) } async function SASL() { + nonce = (await crypto.randomBytes(18)).toString('base64') b().p().str('SCRAM-SHA-256' + b.N) const i = b.i - nonce = (await crypto.randomBytes(18)).toString('base64') write(b.inc(4).str('n,,n=*,r=' + nonce).i32(b.i - i - 4, i).end()) } @@ -698,12 +701,12 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose serverSignature = (await hmac(await hmac(saltedPassword, 'Server Key'), auth)).toString('base64') + const payload = 'c=biws,r=' + res.r + ',p=' + xor( + clientKey, Buffer.from(await hmac(await sha256(clientKey), auth)) + ).toString('base64') + write( - b().p().str( - 'c=biws,r=' + res.r + ',p=' + xor( - clientKey, Buffer.from(await hmac(await sha256(clientKey), auth)) - ).toString('base64') - ).end() + b().p().str(payload).end() ) } diff --git a/cjs/tests/index.js b/cjs/tests/index.js index cb91c5c5..a787bf9f 100644 --- a/cjs/tests/index.js +++ b/cjs/tests/index.js @@ -2134,7 +2134,7 @@ t('Execute', async() => { t('Cancel running query', async() => { const query = sql`select pg_sleep(2)` - setTimeout(() => query.cancel(), 200) + setTimeout(() => query.cancel(), 500) const error = await query.catch(x => x) return ['57014', error.code] }) diff --git a/deno/src/connection.js b/deno/src/connection.js index 95e73dda..bbdb52a1 100644 --- a/deno/src/connection.js +++ b/deno/src/connection.js @@ -659,27 +659,30 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose /* c8 ignore next 5 */ async function AuthenticationCleartextPassword() { + const payload = await Pass() write( - b().p().str(await Pass()).z(1).end() + b().p().str(payload).z(1).end() ) } async function AuthenticationMD5Password(x) { - write( - b().p().str( - 'md5' + - (await md5(Buffer.concat([ + const payload = 'md5' + ( + await md5( + Buffer.concat([ Buffer.from(await md5((await Pass()) + user)), x.subarray(9) - ]))) - ).z(1).end() + ]) + ) + ) + write( + b().p().str(payload).z(1).end() ) } async function SASL() { + nonce = (await crypto.randomBytes(18)).toString('base64') b().p().str('SCRAM-SHA-256' + b.N) const i = b.i - nonce = (await crypto.randomBytes(18)).toString('base64') write(b.inc(4).str('n,,n=*,r=' + nonce).i32(b.i - i - 4, i).end()) } @@ -701,12 +704,12 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose serverSignature = (await hmac(await hmac(saltedPassword, 'Server Key'), auth)).toString('base64') + const payload = 'c=biws,r=' + res.r + ',p=' + xor( + clientKey, Buffer.from(await hmac(await sha256(clientKey), auth)) + ).toString('base64') + write( - b().p().str( - 'c=biws,r=' + res.r + ',p=' + xor( - clientKey, Buffer.from(await hmac(await sha256(clientKey), auth)) - ).toString('base64') - ).end() + b().p().str(payload).end() ) } diff --git a/deno/tests/index.js b/deno/tests/index.js index 08a0c023..d8fcbf36 100644 --- a/deno/tests/index.js +++ b/deno/tests/index.js @@ -2136,7 +2136,7 @@ t('Execute', async() => { t('Cancel running query', async() => { const query = sql`select pg_sleep(2)` - setTimeout(() => query.cancel(), 200) + setTimeout(() => query.cancel(), 500) const error = await query.catch(x => x) return ['57014', error.code] }) diff --git a/tests/index.js b/tests/index.js index 499b3fbd..c28f7626 100644 --- a/tests/index.js +++ b/tests/index.js @@ -2134,7 +2134,7 @@ t('Execute', async() => { t('Cancel running query', async() => { const query = sql`select pg_sleep(2)` - setTimeout(() => query.cancel(), 200) + setTimeout(() => query.cancel(), 500) const error = await query.catch(x => x) return ['57014', error.code] }) From 55186d162a66ce7a6cd470cc6b0a78f9244c501f Mon Sep 17 00:00:00 2001 From: Miles Date: Thu, 26 Oct 2023 15:50:30 -0700 Subject: [PATCH 27/88] Documentation fixes & additions (#699) * Update README.md Add missing awaits and describe dynamic password support. * Add ESM dynamic imports to docs * Update docs transaction example * Minor doc formatting fix --- README.md | 73 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 39678273..07d24d9a 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,14 @@ async function insertUser({ name, age }) { } ``` +#### ESM dynamic imports + +The library can be used with ESM dynamic imports as well as shown here. + +```js +const { default: postgres } = await import('postgres') +``` + ## Table of Contents * [Connection](#connection) @@ -158,7 +166,7 @@ const users = await sql` ```js const columns = ['name', 'age'] -sql` +await sql` select ${ sql(columns) } from users @@ -211,13 +219,13 @@ const users = [{ age: 80 }] -sql`insert into users ${ sql(users, 'name', 'age') }` +await sql`insert into users ${ sql(users, 'name', 'age') }` // Is translated to: insert into users ("name", "age") values ($1, $2), ($3, $4) // Here you can also omit column names which will use object keys as columns -sql`insert into users ${ sql(users) }` +await sql`insert into users ${ sql(users) }` // Which results in: insert into users ("name", "age") values ($1, $2), ($3, $4) @@ -261,7 +269,7 @@ const users = [ [2, 'Jane', 27], ] -sql` +await sql` update users set name = update_data.name, (age = update_data.age)::int from (values ${sql(users)}) as update_data (id, name, age) where users.id = (update_data.id)::int @@ -300,7 +308,7 @@ const olderThan = x => sql`and age > ${ x }` const filterAge = true -sql` +await sql` select * from users @@ -318,7 +326,7 @@ select * from users where name is not null and age > 50 ### Dynamic filters ```js -sql` +await sql` select * from users ${ @@ -339,7 +347,7 @@ Using keywords or calling functions dynamically is also possible by using ``` sq ```js const date = null -sql` +await sql` update users set updated_at = ${ date || sql`now()` } ` @@ -353,7 +361,7 @@ Dynamic identifiers like table names and column names is also supported like so: const table = 'users' , column = 'id' -sql` +await sql` select ${ sql(column) } from ${ sql(table) } ` @@ -367,10 +375,10 @@ Here's a quick oversight over all the ways to do interpolation in a query templa | Interpolation syntax | Usage | Example | | ------------- | ------------- | ------------- | -| `${ sql`` }` | for keywords or sql fragments | ``sql`SELECT * FROM users ${sql`order by age desc` }` `` | -| `${ sql(string) }` | for identifiers | ``sql`SELECT * FROM ${sql('table_name')` `` | -| `${ sql([] or {}, ...) }` | for helpers | ``sql`INSERT INTO users ${sql({ name: 'Peter'})}` `` | -| `${ 'somevalue' }` | for values | ``sql`SELECT * FROM users WHERE age = ${42}` `` | +| `${ sql`` }` | for keywords or sql fragments | ``await sql`SELECT * FROM users ${sql`order by age desc` }` `` | +| `${ sql(string) }` | for identifiers | ``await sql`SELECT * FROM ${sql('table_name')` `` | +| `${ sql([] or {}, ...) }` | for helpers | ``await sql`INSERT INTO users ${sql({ name: 'Peter'})}` `` | +| `${ 'somevalue' }` | for values | ``await sql`SELECT * FROM users WHERE age = ${42}` `` | ## Advanced query methods @@ -450,7 +458,7 @@ await sql` Rather than executing a given query, `.describe` will return information utilized in the query process. This information can include the query identifier, column types, etc. This is useful for debugging and analyzing your Postgres queries. Furthermore, **`.describe` will give you access to the final generated query string that would be executed.** - + ### Rows as Array of Values #### ```sql``.values()``` @@ -477,7 +485,7 @@ const result = await sql.file('query.sql', ['Murray', 68]) ### Multiple statements in one query #### ```await sql``.simple()``` -The postgres wire protocol supports ["simple"](https://www.postgresql.org/docs/current/protocol-flow.html#id-1.10.6.7.4) and ["extended"](https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-FLOW-EXT-QUERY) queries. "simple" queries supports multiple statements, but does not support any dynamic parameters. "extended" queries support parameters but only one statement. To use "simple" queries you can use +The postgres wire protocol supports ["simple"](https://www.postgresql.org/docs/current/protocol-flow.html#id-1.10.6.7.4) and ["extended"](https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-FLOW-EXT-QUERY) queries. "simple" queries supports multiple statements, but does not support any dynamic parameters. "extended" queries support parameters but only one statement. To use "simple" queries you can use ```sql``.simple()```. That will create it as a simple query. ```js @@ -519,8 +527,8 @@ await pipeline(readableStream, createWriteStream('output.tsv')) ```js const readableStream = await sql` copy ( - select name, age - from users + select name, age + from users where age = 68 ) to stdout `.readable() @@ -559,7 +567,7 @@ If you know what you're doing, you can use `unsafe` to pass any string you'd lik ```js sql.unsafe('select ' + danger + ' from users where id = ' + dragons) ``` - + You can also nest `sql.unsafe` within a safe `sql` expression. This is useful if only part of your fraction has unsafe elements. ```js @@ -599,7 +607,7 @@ const [user, account] = await sql.begin(async sql => { ) values ( 'Murray' ) - returning * + returning * ` const [account] = await sql` @@ -608,7 +616,7 @@ const [user, account] = await sql.begin(async sql => { ) values ( ${ user.user_id } ) - returning * + returning * ` return [user, account] @@ -676,7 +684,7 @@ sql.begin('read write', async sql => { 'Murray' ) ` - + await sql.prepare('tx1') }) ``` @@ -736,7 +744,7 @@ console.log(data) // [ { a_test: 1 } ] ### Transform `undefined` Values -By default, Postgres.js will throw the error `UNDEFINED_VALUE: Undefined values are not allowed` when undefined values are passed +By default, Postgres.js will throw the error `UNDEFINED_VALUE: Undefined values are not allowed` when undefined values are passed ```js // Transform the column names to and from camel case @@ -817,7 +825,7 @@ The optional `onlisten` method is great to use for a very simply queue mechanism ```js await sql.listen( - 'jobs', + 'jobs', (x) => run(JSON.parse(x)), ( ) => sql`select unfinished_jobs()`.forEach(run) ) @@ -850,7 +858,7 @@ CREATE PUBLICATION alltables FOR ALL TABLES const sql = postgres({ publications: 'alltables' }) const { unsubscribe } = await sql.subscribe( - 'insert:events', + 'insert:events', (row, { command, relation, key, old }) => { // Callback function for each row change // tell about new event row over eg. websockets or do something else @@ -986,6 +994,19 @@ const sql = postgres('postgres://username:password@host:port/database', { Note that `max_lifetime = 60 * (30 + Math.random() * 30)` by default. This resolves to an interval between 45 and 90 minutes to optimize for the benefits of prepared statements **and** working nicely with Linux's OOM killer. +### Dynamic passwords + +When clients need to use alternative authentication schemes such as access tokens or connections to databases with rotating passwords, provide either a synchronous or asynchronous function that will resolve the dynamic password value at connection time. + +```js +const sql = postgres(url, { + // Other connection config + ... + // Password function for the database user + password : async () => await signer.getAuthToken(), +}) +``` + ### SSL Although [vulnerable to MITM attacks](https://security.stackexchange.com/a/229297/174913), a common configuration for the `ssl` option for some cloud providers is to set `rejectUnauthorized` to `false` (if `NODE_ENV` is `production`): @@ -1144,7 +1165,7 @@ const sql = postgres({ }) // Now you can use sql.typed.rect() as specified above -const [custom] = sql` +const [custom] = await sql` insert into rectangles ( name, rect @@ -1174,8 +1195,8 @@ const sql = postgres({ const ssh = new ssh2.Client() ssh .on('error', reject) - .on('ready', () => - ssh.forwardOut('127.0.0.1', 12345, host, port, + .on('ready', () => + ssh.forwardOut('127.0.0.1', 12345, host, port, (err, socket) => err ? reject(err) : resolve(socket) ) ) From 0bee4c30a2c98bb27e43bdd0a3161e3174b187b0 Mon Sep 17 00:00:00 2001 From: Nick Randall Date: Thu, 26 Oct 2023 15:52:01 -0700 Subject: [PATCH 28/88] adding support for sslrootcert option in connection string (#690) * adding support for sslrootcert option in connection string * Update index.js --- src/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/index.js b/src/index.js index ff990586..7ed05d8c 100644 --- a/src/index.js +++ b/src/index.js @@ -437,6 +437,7 @@ function parseOptions(a, b) { o.no_prepare && (o.prepare = false) query.sslmode && (query.ssl = query.sslmode, delete query.sslmode) 'timeout' in o && (console.log('The timeout option is deprecated, use idle_timeout instead'), o.idle_timeout = o.timeout) // eslint-disable-line + query.sslrootcert === 'system' && (query.ssl = 'verify-full') const ints = ['idle_timeout', 'connect_timeout', 'max_lifetime', 'max_pipeline', 'backoff', 'keep_alive'] const defaults = { From f2fb819de4078ec6ff3e4dbf6c5dc117c2d5b0a0 Mon Sep 17 00:00:00 2001 From: Pyrolistical Date: Thu, 26 Oct 2023 15:53:42 -0700 Subject: [PATCH 29/88] Keep query error instead of creating creating new object (#698) * Keep query error instead of creating creating new object fixes #696 * Enumerate properties only if debug * Fixed typo * Fixed styling --- src/connection.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/connection.js b/src/connection.js index 1135189f..389d4a7d 100644 --- a/src/connection.js +++ b/src/connection.js @@ -385,13 +385,14 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } function queryError(query, err) { - query.reject(Object.create(err, { + Object.defineProperties(err, { stack: { value: err.stack + query.origin.replace(/.*\n/, '\n'), enumerable: options.debug }, query: { value: query.string, enumerable: options.debug }, parameters: { value: query.parameters, enumerable: options.debug }, args: { value: query.args, enumerable: options.debug }, types: { value: query.statement && query.statement.types, enumerable: options.debug } - })) + }) + query.reject(err) } function end() { From ca2754cf484108f50bc0183849490111b3f28b7c Mon Sep 17 00:00:00 2001 From: Martin Kubliniak Date: Fri, 27 Oct 2023 01:17:40 +0200 Subject: [PATCH 30/88] Add common parameter names to ConnectionParameters TS type (#707) --- README.md | 2 +- deno/README.md | 2 +- deno/types/index.d.ts | 12 +++++++++++- types/index.d.ts | 12 +++++++++++- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 07d24d9a..da002cca 100644 --- a/README.md +++ b/README.md @@ -983,7 +983,7 @@ const sql = postgres('postgres://username:password@host:port/database', { }, connection : { application_name : 'postgres.js', // Default application_name - ... // Other connection parameters + ... // Other connection parameters, see https://www.postgresql.org/docs/current/runtime-config-client.html }, target_session_attrs : null, // Use 'read-write' with multiple hosts to // ensure only connecting to primary diff --git a/deno/README.md b/deno/README.md index 19fd0993..d80fea5f 100644 --- a/deno/README.md +++ b/deno/README.md @@ -971,7 +971,7 @@ const sql = postgres('postgres://username:password@host:port/database', { }, connection : { application_name : 'postgres.js', // Default application_name - ... // Other connection parameters + ... // Other connection parameters, see https://www.postgresql.org/docs/current/runtime-config-client.html }, target_session_attrs : null, // Use 'read-write' with multiple hosts to // ensure only connecting to primary diff --git a/deno/types/index.d.ts b/deno/types/index.d.ts index 215d5b62..6f96fe97 100644 --- a/deno/types/index.d.ts +++ b/deno/types/index.d.ts @@ -331,8 +331,18 @@ declare namespace postgres { * @default 'postgres.js' */ application_name: string; + default_transaction_isolation: 'read uncommitted' | 'read committed' | 'repeatable read' | 'serializable', + default_transaction_read_only: boolean, + default_transaction_deferrable: boolean, + statement_timeout: number, + lock_timeout: number, + idle_in_transaction_session_timeout: number, + idle_session_timeout: number, + DateStyle: string, + IntervalStyle: string, + TimeZone: string, /** Other connection parameters */ - [name: string]: string; + [name: string]: string | number | boolean; } interface Options> extends Partial> { diff --git a/types/index.d.ts b/types/index.d.ts index 8dacd9c4..78d559ef 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -329,8 +329,18 @@ declare namespace postgres { * @default 'postgres.js' */ application_name: string; + default_transaction_isolation: 'read uncommitted' | 'read committed' | 'repeatable read' | 'serializable', + default_transaction_read_only: boolean, + default_transaction_deferrable: boolean, + statement_timeout: number, + lock_timeout: number, + idle_in_transaction_session_timeout: number, + idle_session_timeout: number, + DateStyle: string, + IntervalStyle: string, + TimeZone: string, /** Other connection parameters */ - [name: string]: string; + [name: string]: string | number | boolean; } interface Options> extends Partial> { From 788c8191b0885d4feb073d862172c9e51375414f Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Fri, 27 Oct 2023 17:45:19 +0200 Subject: [PATCH 31/88] Ensure transactions throw if connection is closed while there is no active query - fixes #658 --- src/connection.js | 2 +- src/index.js | 8 ++++++-- tests/index.js | 10 ++++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/connection.js b/src/connection.js index 389d4a7d..a6825105 100644 --- a/src/connection.js +++ b/src/connection.js @@ -441,7 +441,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose closedDate = performance.now() hadError && options.shared.retries++ delay = (typeof backoff === 'function' ? backoff(options.shared.retries) : backoff) * 1000 - onclose(connection) + onclose(connection, Errors.connection('CONNECTION_CLOSED', options, socket)) } /* Handlers */ diff --git a/src/index.js b/src/index.js index 7ed05d8c..0573e2bc 100644 --- a/src/index.js +++ b/src/index.js @@ -239,7 +239,10 @@ function Postgres(a, b) { try { await sql.unsafe('begin ' + options.replace(/[^a-z ]/ig, ''), [], { onexecute }).execute() - return await scope(connection, fn) + return await Promise.race([ + scope(connection, fn), + new Promise((_, reject) => connection.onclose = reject) + ]) } catch (error) { throw error } @@ -414,9 +417,10 @@ function Postgres(a, b) { : move(c, full) } - function onclose(c) { + function onclose(c, e) { move(c, closed) c.reserved = null + c.onclose && (c.onclose(e), c.onclose = null) options.onclose && options.onclose(c.id) queries.length && connect(c, queries.shift()) } diff --git a/tests/index.js b/tests/index.js index c28f7626..86100399 100644 --- a/tests/index.js +++ b/tests/index.js @@ -2348,6 +2348,16 @@ t('Ensure reconnect after max_lifetime with transactions', { timeout: 5 }, async return [true, true] }) + +t('Ensure transactions throw if connection is closed dwhile there is no query', async() => { + const x = await sql.begin(async() => { + setTimeout(() => sql.end({ timeout: 0 }), 10) + await new Promise(r => setTimeout(r, 200)) + return sql`select 1` + }).catch(x => x) + return ['CONNECTION_CLOSED', x.code] +}) + t('Custom socket', {}, async() => { let result const sql = postgres({ From 26b23c170b198f797b6476621dba492c6a9d6ba6 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Fri, 27 Oct 2023 17:52:45 +0200 Subject: [PATCH 32/88] Fix test --- tests/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/index.js b/tests/index.js index 86100399..e47cb534 100644 --- a/tests/index.js +++ b/tests/index.js @@ -2350,6 +2350,7 @@ t('Ensure reconnect after max_lifetime with transactions', { timeout: 5 }, async t('Ensure transactions throw if connection is closed dwhile there is no query', async() => { + const sql = postgres(options) const x = await sql.begin(async() => { setTimeout(() => sql.end({ timeout: 0 }), 10) await new Promise(r => setTimeout(r, 200)) From aa3d13ea36b9865f21c4b6d843cbfc03b3665de8 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Fri, 27 Oct 2023 17:54:15 +0200 Subject: [PATCH 33/88] build --- cf/src/connection.js | 36 ++++++++++++--------- cf/src/index.js | 9 ++++-- cjs/src/connection.js | 7 ++-- cjs/src/index.js | 9 ++++-- cjs/tests/index.js | 11 +++++++ deno/README.md | 73 +++++++++++++++++++++++++++--------------- deno/src/connection.js | 7 ++-- deno/src/index.js | 9 ++++-- deno/tests/index.js | 11 +++++++ 9 files changed, 118 insertions(+), 54 deletions(-) diff --git a/cf/src/connection.js b/cf/src/connection.js index c09b2720..f06a5f8b 100644 --- a/cf/src/connection.js +++ b/cf/src/connection.js @@ -387,13 +387,14 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } function queryError(query, err) { - query.reject(Object.create(err, { + Object.defineProperties(err, { stack: { value: err.stack + query.origin.replace(/.*\n/, '\n'), enumerable: options.debug }, query: { value: query.string, enumerable: options.debug }, parameters: { value: query.parameters, enumerable: options.debug }, args: { value: query.args, enumerable: options.debug }, types: { value: query.statement && query.statement.types, enumerable: options.debug } - })) + }) + query.reject(err) } function end() { @@ -442,7 +443,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose closedDate = performance.now() hadError && options.shared.retries++ delay = (typeof backoff === 'function' ? backoff(options.shared.retries) : backoff) * 1000 - onclose(connection) + onclose(connection, Errors.connection('CONNECTION_CLOSED', options, socket)) } /* Handlers */ @@ -658,27 +659,30 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose /* c8 ignore next 5 */ async function AuthenticationCleartextPassword() { + const payload = await Pass() write( - b().p().str(await Pass()).z(1).end() + b().p().str(payload).z(1).end() ) } async function AuthenticationMD5Password(x) { - write( - b().p().str( - 'md5' + - (await md5(Buffer.concat([ + const payload = 'md5' + ( + await md5( + Buffer.concat([ Buffer.from(await md5((await Pass()) + user)), x.subarray(9) - ]))) - ).z(1).end() + ]) + ) + ) + write( + b().p().str(payload).z(1).end() ) } async function SASL() { + nonce = (await crypto.randomBytes(18)).toString('base64') b().p().str('SCRAM-SHA-256' + b.N) const i = b.i - nonce = (await crypto.randomBytes(18)).toString('base64') write(b.inc(4).str('n,,n=*,r=' + nonce).i32(b.i - i - 4, i).end()) } @@ -700,12 +704,12 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose serverSignature = (await hmac(await hmac(saltedPassword, 'Server Key'), auth)).toString('base64') + const payload = 'c=biws,r=' + res.r + ',p=' + xor( + clientKey, Buffer.from(await hmac(await sha256(clientKey), auth)) + ).toString('base64') + write( - b().p().str( - 'c=biws,r=' + res.r + ',p=' + xor( - clientKey, Buffer.from(await hmac(await sha256(clientKey), auth)) - ).toString('base64') - ).end() + b().p().str(payload).end() ) } diff --git a/cf/src/index.js b/cf/src/index.js index 0c74f5cf..d24e9f9c 100644 --- a/cf/src/index.js +++ b/cf/src/index.js @@ -240,7 +240,10 @@ function Postgres(a, b) { try { await sql.unsafe('begin ' + options.replace(/[^a-z ]/ig, ''), [], { onexecute }).execute() - return await scope(connection, fn) + return await Promise.race([ + scope(connection, fn), + new Promise((_, reject) => connection.onclose = reject) + ]) } catch (error) { throw error } @@ -415,9 +418,10 @@ function Postgres(a, b) { : move(c, full) } - function onclose(c) { + function onclose(c, e) { move(c, closed) c.reserved = null + c.onclose && (c.onclose(e), c.onclose = null) options.onclose && options.onclose(c.id) queries.length && connect(c, queries.shift()) } @@ -438,6 +442,7 @@ function parseOptions(a, b) { o.no_prepare && (o.prepare = false) query.sslmode && (query.ssl = query.sslmode, delete query.sslmode) 'timeout' in o && (console.log('The timeout option is deprecated, use idle_timeout instead'), o.idle_timeout = o.timeout) // eslint-disable-line + query.sslrootcert === 'system' && (query.ssl = 'verify-full') const ints = ['idle_timeout', 'connect_timeout', 'max_lifetime', 'max_pipeline', 'backoff', 'keep_alive'] const defaults = { diff --git a/cjs/src/connection.js b/cjs/src/connection.js index c07d3027..b295958a 100644 --- a/cjs/src/connection.js +++ b/cjs/src/connection.js @@ -385,13 +385,14 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } function queryError(query, err) { - query.reject(Object.create(err, { + Object.defineProperties(err, { stack: { value: err.stack + query.origin.replace(/.*\n/, '\n'), enumerable: options.debug }, query: { value: query.string, enumerable: options.debug }, parameters: { value: query.parameters, enumerable: options.debug }, args: { value: query.args, enumerable: options.debug }, types: { value: query.statement && query.statement.types, enumerable: options.debug } - })) + }) + query.reject(err) } function end() { @@ -440,7 +441,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose closedDate = performance.now() hadError && options.shared.retries++ delay = (typeof backoff === 'function' ? backoff(options.shared.retries) : backoff) * 1000 - onclose(connection) + onclose(connection, Errors.connection('CONNECTION_CLOSED', options, socket)) } /* Handlers */ diff --git a/cjs/src/index.js b/cjs/src/index.js index 698b05d4..40ac2c18 100644 --- a/cjs/src/index.js +++ b/cjs/src/index.js @@ -239,7 +239,10 @@ function Postgres(a, b) { try { await sql.unsafe('begin ' + options.replace(/[^a-z ]/ig, ''), [], { onexecute }).execute() - return await scope(connection, fn) + return await Promise.race([ + scope(connection, fn), + new Promise((_, reject) => connection.onclose = reject) + ]) } catch (error) { throw error } @@ -414,9 +417,10 @@ function Postgres(a, b) { : move(c, full) } - function onclose(c) { + function onclose(c, e) { move(c, closed) c.reserved = null + c.onclose && (c.onclose(e), c.onclose = null) options.onclose && options.onclose(c.id) queries.length && connect(c, queries.shift()) } @@ -437,6 +441,7 @@ function parseOptions(a, b) { o.no_prepare && (o.prepare = false) query.sslmode && (query.ssl = query.sslmode, delete query.sslmode) 'timeout' in o && (console.log('The timeout option is deprecated, use idle_timeout instead'), o.idle_timeout = o.timeout) // eslint-disable-line + query.sslrootcert === 'system' && (query.ssl = 'verify-full') const ints = ['idle_timeout', 'connect_timeout', 'max_lifetime', 'max_pipeline', 'backoff', 'keep_alive'] const defaults = { diff --git a/cjs/tests/index.js b/cjs/tests/index.js index a787bf9f..ef70c4ab 100644 --- a/cjs/tests/index.js +++ b/cjs/tests/index.js @@ -2348,6 +2348,17 @@ t('Ensure reconnect after max_lifetime with transactions', { timeout: 5 }, async return [true, true] }) + +t('Ensure transactions throw if connection is closed dwhile there is no query', async() => { + const sql = postgres(options) + const x = await sql.begin(async() => { + setTimeout(() => sql.end({ timeout: 0 }), 10) + await new Promise(r => setTimeout(r, 200)) + return sql`select 1` + }).catch(x => x) + return ['CONNECTION_CLOSED', x.code] +}) + t('Custom socket', {}, async() => { let result const sql = postgres({ diff --git a/deno/README.md b/deno/README.md index d80fea5f..0fc569bb 100644 --- a/deno/README.md +++ b/deno/README.md @@ -58,6 +58,14 @@ async function insertUser({ name, age }) { } ``` +#### ESM dynamic imports + +The library can be used with ESM dynamic imports as well as shown here. + +```js +const { default: postgres } = await import('postgres') +``` + ## Table of Contents * [Connection](#connection) @@ -154,7 +162,7 @@ const users = await sql` ```js const columns = ['name', 'age'] -sql` +await sql` select ${ sql(columns) } from users @@ -207,13 +215,13 @@ const users = [{ age: 80 }] -sql`insert into users ${ sql(users, 'name', 'age') }` +await sql`insert into users ${ sql(users, 'name', 'age') }` // Is translated to: insert into users ("name", "age") values ($1, $2), ($3, $4) // Here you can also omit column names which will use object keys as columns -sql`insert into users ${ sql(users) }` +await sql`insert into users ${ sql(users) }` // Which results in: insert into users ("name", "age") values ($1, $2), ($3, $4) @@ -257,7 +265,7 @@ const users = [ [2, 'Jane', 27], ] -sql` +await sql` update users set name = update_data.name, (age = update_data.age)::int from (values ${sql(users)}) as update_data (id, name, age) where users.id = (update_data.id)::int @@ -296,7 +304,7 @@ const olderThan = x => sql`and age > ${ x }` const filterAge = true -sql` +await sql` select * from users @@ -314,7 +322,7 @@ select * from users where name is not null and age > 50 ### Dynamic filters ```js -sql` +await sql` select * from users ${ @@ -335,7 +343,7 @@ Using keywords or calling functions dynamically is also possible by using ``` sq ```js const date = null -sql` +await sql` update users set updated_at = ${ date || sql`now()` } ` @@ -349,7 +357,7 @@ Dynamic identifiers like table names and column names is also supported like so: const table = 'users' , column = 'id' -sql` +await sql` select ${ sql(column) } from ${ sql(table) } ` @@ -363,10 +371,10 @@ Here's a quick oversight over all the ways to do interpolation in a query templa | Interpolation syntax | Usage | Example | | ------------- | ------------- | ------------- | -| `${ sql`` }` | for keywords or sql fragments | ``sql`SELECT * FROM users ${sql`order by age desc` }` `` | -| `${ sql(string) }` | for identifiers | ``sql`SELECT * FROM ${sql('table_name')` `` | -| `${ sql([] or {}, ...) }` | for helpers | ``sql`INSERT INTO users ${sql({ name: 'Peter'})}` `` | -| `${ 'somevalue' }` | for values | ``sql`SELECT * FROM users WHERE age = ${42}` `` | +| `${ sql`` }` | for keywords or sql fragments | ``await sql`SELECT * FROM users ${sql`order by age desc` }` `` | +| `${ sql(string) }` | for identifiers | ``await sql`SELECT * FROM ${sql('table_name')` `` | +| `${ sql([] or {}, ...) }` | for helpers | ``await sql`INSERT INTO users ${sql({ name: 'Peter'})}` `` | +| `${ 'somevalue' }` | for values | ``await sql`SELECT * FROM users WHERE age = ${42}` `` | ## Advanced query methods @@ -446,7 +454,7 @@ await sql` Rather than executing a given query, `.describe` will return information utilized in the query process. This information can include the query identifier, column types, etc. This is useful for debugging and analyzing your Postgres queries. Furthermore, **`.describe` will give you access to the final generated query string that would be executed.** - + ### Rows as Array of Values #### ```sql``.values()``` @@ -473,7 +481,7 @@ const result = await sql.file('query.sql', ['Murray', 68]) ### Multiple statements in one query #### ```await sql``.simple()``` -The postgres wire protocol supports ["simple"](https://www.postgresql.org/docs/current/protocol-flow.html#id-1.10.6.7.4) and ["extended"](https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-FLOW-EXT-QUERY) queries. "simple" queries supports multiple statements, but does not support any dynamic parameters. "extended" queries support parameters but only one statement. To use "simple" queries you can use +The postgres wire protocol supports ["simple"](https://www.postgresql.org/docs/current/protocol-flow.html#id-1.10.6.7.4) and ["extended"](https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-FLOW-EXT-QUERY) queries. "simple" queries supports multiple statements, but does not support any dynamic parameters. "extended" queries support parameters but only one statement. To use "simple" queries you can use ```sql``.simple()```. That will create it as a simple query. ```js @@ -515,8 +523,8 @@ await pipeline(readableStream, createWriteStream('output.tsv')) ```js const readableStream = await sql` copy ( - select name, age - from users + select name, age + from users where age = 68 ) to stdout `.readable() @@ -555,7 +563,7 @@ If you know what you're doing, you can use `unsafe` to pass any string you'd lik ```js sql.unsafe('select ' + danger + ' from users where id = ' + dragons) ``` - + You can also nest `sql.unsafe` within a safe `sql` expression. This is useful if only part of your fraction has unsafe elements. ```js @@ -595,7 +603,7 @@ const [user, account] = await sql.begin(async sql => { ) values ( 'Murray' ) - returning * + returning * ` const [account] = await sql` @@ -604,7 +612,7 @@ const [user, account] = await sql.begin(async sql => { ) values ( ${ user.user_id } ) - returning * + returning * ` return [user, account] @@ -672,7 +680,7 @@ sql.begin('read write', async sql => { 'Murray' ) ` - + await sql.prepare('tx1') }) ``` @@ -732,7 +740,7 @@ console.log(data) // [ { a_test: 1 } ] ### Transform `undefined` Values -By default, Postgres.js will throw the error `UNDEFINED_VALUE: Undefined values are not allowed` when undefined values are passed +By default, Postgres.js will throw the error `UNDEFINED_VALUE: Undefined values are not allowed` when undefined values are passed ```js // Transform the column names to and from camel case @@ -813,7 +821,7 @@ The optional `onlisten` method is great to use for a very simply queue mechanism ```js await sql.listen( - 'jobs', + 'jobs', (x) => run(JSON.parse(x)), ( ) => sql`select unfinished_jobs()`.forEach(run) ) @@ -846,7 +854,7 @@ CREATE PUBLICATION alltables FOR ALL TABLES const sql = postgres({ publications: 'alltables' }) const { unsubscribe } = await sql.subscribe( - 'insert:events', + 'insert:events', (row, { command, relation, key, old }) => { // Callback function for each row change // tell about new event row over eg. websockets or do something else @@ -982,6 +990,19 @@ const sql = postgres('postgres://username:password@host:port/database', { Note that `max_lifetime = 60 * (30 + Math.random() * 30)` by default. This resolves to an interval between 45 and 90 minutes to optimize for the benefits of prepared statements **and** working nicely with Linux's OOM killer. +### Dynamic passwords + +When clients need to use alternative authentication schemes such as access tokens or connections to databases with rotating passwords, provide either a synchronous or asynchronous function that will resolve the dynamic password value at connection time. + +```js +const sql = postgres(url, { + // Other connection config + ... + // Password function for the database user + password : async () => await signer.getAuthToken(), +}) +``` + ### SSL Although [vulnerable to MITM attacks](https://security.stackexchange.com/a/229297/174913), a common configuration for the `ssl` option for some cloud providers is to set `rejectUnauthorized` to `false` (if `NODE_ENV` is `production`): @@ -1140,7 +1161,7 @@ const sql = postgres({ }) // Now you can use sql.typed.rect() as specified above -const [custom] = sql` +const [custom] = await sql` insert into rectangles ( name, rect @@ -1170,8 +1191,8 @@ const sql = postgres({ const ssh = new ssh2.Client() ssh .on('error', reject) - .on('ready', () => - ssh.forwardOut('127.0.0.1', 12345, host, port, + .on('ready', () => + ssh.forwardOut('127.0.0.1', 12345, host, port, (err, socket) => err ? reject(err) : resolve(socket) ) ) diff --git a/deno/src/connection.js b/deno/src/connection.js index bbdb52a1..bc4d231c 100644 --- a/deno/src/connection.js +++ b/deno/src/connection.js @@ -388,13 +388,14 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } function queryError(query, err) { - query.reject(Object.create(err, { + Object.defineProperties(err, { stack: { value: err.stack + query.origin.replace(/.*\n/, '\n'), enumerable: options.debug }, query: { value: query.string, enumerable: options.debug }, parameters: { value: query.parameters, enumerable: options.debug }, args: { value: query.args, enumerable: options.debug }, types: { value: query.statement && query.statement.types, enumerable: options.debug } - })) + }) + query.reject(err) } function end() { @@ -443,7 +444,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose closedDate = performance.now() hadError && options.shared.retries++ delay = (typeof backoff === 'function' ? backoff(options.shared.retries) : backoff) * 1000 - onclose(connection) + onclose(connection, Errors.connection('CONNECTION_CLOSED', options, socket)) } /* Handlers */ diff --git a/deno/src/index.js b/deno/src/index.js index fada05ae..3bbdf2ba 100644 --- a/deno/src/index.js +++ b/deno/src/index.js @@ -240,7 +240,10 @@ function Postgres(a, b) { try { await sql.unsafe('begin ' + options.replace(/[^a-z ]/ig, ''), [], { onexecute }).execute() - return await scope(connection, fn) + return await Promise.race([ + scope(connection, fn), + new Promise((_, reject) => connection.onclose = reject) + ]) } catch (error) { throw error } @@ -415,9 +418,10 @@ function Postgres(a, b) { : move(c, full) } - function onclose(c) { + function onclose(c, e) { move(c, closed) c.reserved = null + c.onclose && (c.onclose(e), c.onclose = null) options.onclose && options.onclose(c.id) queries.length && connect(c, queries.shift()) } @@ -438,6 +442,7 @@ function parseOptions(a, b) { o.no_prepare && (o.prepare = false) query.sslmode && (query.ssl = query.sslmode, delete query.sslmode) 'timeout' in o && (console.log('The timeout option is deprecated, use idle_timeout instead'), o.idle_timeout = o.timeout) // eslint-disable-line + query.sslrootcert === 'system' && (query.ssl = 'verify-full') const ints = ['idle_timeout', 'connect_timeout', 'max_lifetime', 'max_pipeline', 'backoff', 'keep_alive'] const defaults = { diff --git a/deno/tests/index.js b/deno/tests/index.js index d8fcbf36..dc78c2c8 100644 --- a/deno/tests/index.js +++ b/deno/tests/index.js @@ -2350,6 +2350,17 @@ t('Ensure reconnect after max_lifetime with transactions', { timeout: 5 }, async return [true, true] }) + +t('Ensure transactions throw if connection is closed dwhile there is no query', async() => { + const sql = postgres(options) + const x = await sql.begin(async() => { + setTimeout(() => sql.end({ timeout: 0 }), 10) + await new Promise(r => setTimeout(r, 200)) + return sql`select 1` + }).catch(x => x) + return ['CONNECTION_CLOSED', x.code] +}) + t('Custom socket', {}, async() => { let result const sql = postgres({ From 42a81d1651659954b69ff2b1b07bf3893560dede Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Fri, 27 Oct 2023 17:56:48 +0200 Subject: [PATCH 34/88] Add node 21 to tests --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 85a859ff..6da2dbd1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - node: ['12', '14', '16', '18', '20'] + node: ['12', '14', '16', '18', '20', '21'] postgres: ['12', '13', '14', '15', '16'] runs-on: ubuntu-latest services: From b25274c546d562f24ea2c60b030acb23f51d4400 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Fri, 27 Oct 2023 17:58:18 +0200 Subject: [PATCH 35/88] 3.4.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e8a552d1..34802d6c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "postgres", - "version": "3.4.1", + "version": "3.4.2", "description": "Fastest full featured PostgreSQL client for Node.js", "type": "module", "module": "src/index.js", From c084a1cf0ffd5aeaf9388ef0c84d0da28fca24b5 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Thu, 2 Nov 2023 08:28:13 +0100 Subject: [PATCH 36/88] Ensure reserved connections are initialized properly - fixes #718 --- src/connection.js | 11 +++++++---- tests/index.js | 11 +++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/connection.js b/src/connection.js index a6825105..7d97a4b7 100644 --- a/src/connection.js +++ b/src/connection.js @@ -109,7 +109,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose queue: queues.closed, idleTimer, connect(query) { - initial = query + initial = query || true reconnect() }, terminate, @@ -533,11 +533,14 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose return terminate() } - if (needsTypes) + if (needsTypes) { + initial === true && (initial = null) return fetchArrayTypes() + } - execute(initial) - options.shared.retries = retries = initial = 0 + initial !== true && execute(initial) + options.shared.retries = retries = 0 + initial = null return } diff --git a/tests/index.js b/tests/index.js index e47cb534..cd08370a 100644 --- a/tests/index.js +++ b/tests/index.js @@ -2543,3 +2543,14 @@ t('reserve connection', async() => { xs.map(x => x.x).join('') ] }) + +t('arrays in reserved connection', async() => { + const reserved = await sql.reserve() + const [{ x }] = await reserved`select array[1, 2, 3] as x` + reserved.release() + + return [ + '123', + x.join('') + ] +}) From 6121a0afc100f968b50112b225a8e55660687160 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Thu, 2 Nov 2023 08:44:19 +0100 Subject: [PATCH 37/88] build --- cf/src/connection.js | 11 +++++++---- cjs/src/connection.js | 11 +++++++---- cjs/tests/index.js | 11 +++++++++++ deno/src/connection.js | 11 +++++++---- deno/tests/index.js | 11 +++++++++++ 5 files changed, 43 insertions(+), 12 deletions(-) diff --git a/cf/src/connection.js b/cf/src/connection.js index f06a5f8b..ab977ca8 100644 --- a/cf/src/connection.js +++ b/cf/src/connection.js @@ -111,7 +111,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose queue: queues.closed, idleTimer, connect(query) { - initial = query + initial = query || true reconnect() }, terminate, @@ -535,11 +535,14 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose return terminate() } - if (needsTypes) + if (needsTypes) { + initial === true && (initial = null) return fetchArrayTypes() + } - execute(initial) - options.shared.retries = retries = initial = 0 + initial !== true && execute(initial) + options.shared.retries = retries = 0 + initial = null return } diff --git a/cjs/src/connection.js b/cjs/src/connection.js index b295958a..425e91cd 100644 --- a/cjs/src/connection.js +++ b/cjs/src/connection.js @@ -109,7 +109,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose queue: queues.closed, idleTimer, connect(query) { - initial = query + initial = query || true reconnect() }, terminate, @@ -533,11 +533,14 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose return terminate() } - if (needsTypes) + if (needsTypes) { + initial === true && (initial = null) return fetchArrayTypes() + } - execute(initial) - options.shared.retries = retries = initial = 0 + initial !== true && execute(initial) + options.shared.retries = retries = 0 + initial = null return } diff --git a/cjs/tests/index.js b/cjs/tests/index.js index ef70c4ab..5aa0ae15 100644 --- a/cjs/tests/index.js +++ b/cjs/tests/index.js @@ -2543,3 +2543,14 @@ t('reserve connection', async() => { xs.map(x => x.x).join('') ] }) + +t('arrays in reserved connection', async() => { + const reserved = await sql.reserve() + const [{ x }] = await reserved`select array[1, 2, 3] as x` + reserved.release() + + return [ + '123', + x.join('') + ] +}) diff --git a/deno/src/connection.js b/deno/src/connection.js index bc4d231c..334b9722 100644 --- a/deno/src/connection.js +++ b/deno/src/connection.js @@ -112,7 +112,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose queue: queues.closed, idleTimer, connect(query) { - initial = query + initial = query || true reconnect() }, terminate, @@ -536,11 +536,14 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose return terminate() } - if (needsTypes) + if (needsTypes) { + initial === true && (initial = null) return fetchArrayTypes() + } - execute(initial) - options.shared.retries = retries = initial = 0 + initial !== true && execute(initial) + options.shared.retries = retries = 0 + initial = null return } diff --git a/deno/tests/index.js b/deno/tests/index.js index dc78c2c8..90d1feeb 100644 --- a/deno/tests/index.js +++ b/deno/tests/index.js @@ -2546,4 +2546,15 @@ t('reserve connection', async() => { ] }) +t('arrays in reserved connection', async() => { + const reserved = await sql.reserve() + const [{ x }] = await reserved`select array[1, 2, 3] as x` + reserved.release() + + return [ + '123', + x.join('') + ] +}) + ;window.addEventListener("unload", () => Deno.exit(process.exitCode)) \ No newline at end of file From 61c4d5b1d840ed1e3e0f8e84556544a33ee04149 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Thu, 2 Nov 2023 08:45:01 +0100 Subject: [PATCH 38/88] 3.4.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 34802d6c..ea500a80 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "postgres", - "version": "3.4.2", + "version": "3.4.3", "description": "Fastest full featured PostgreSQL client for Node.js", "type": "module", "module": "src/index.js", From 6f20a4820c683b33e7670b606d8daf5670f4b973 Mon Sep 17 00:00:00 2001 From: Wack <135170502+wackfx@users.noreply.github.com> Date: Sun, 26 Nov 2023 10:01:37 +0100 Subject: [PATCH 39/88] Patch: Connection stuck after a while (#738) * Update connection.js --- src/connection.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/connection.js b/src/connection.js index 7d97a4b7..a5694183 100644 --- a/src/connection.js +++ b/src/connection.js @@ -429,10 +429,8 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose lifeTimer.cancel() connectTimer.cancel() - if (socket.encrypted) { - socket.removeAllListeners() - socket = null - } + socket.removeAllListeners() + socket = null if (initial) return reconnect() From 3623021f78b2c92d30f86ac96038941c51d93527 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 30 Jan 2024 20:15:35 +0000 Subject: [PATCH 40/88] docs: update Cloudflare Workers instructions --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index da002cca..c2202bcc 100644 --- a/README.md +++ b/README.md @@ -1103,10 +1103,10 @@ export default async fetch(req: Request, env: Env, ctx: ExecutionContext) { } ``` -In `wrangler.toml` you will need to enable `node_compat` to allow Postgres.js to operate in the Workers environment: +In `wrangler.toml` you will need to enable the `nodejs_compat` compatibility flag to allow Postgres.js to operate in the Workers environment: ```toml -node_compat = true # required for database drivers to function +compatibility_flags = ["nodejs_compat"] ``` ### Auto fetching of array types From cd02af83bdc6fd6d9801d793825f0bb0af36f074 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Sat, 17 Feb 2024 11:29:28 +0100 Subject: [PATCH 41/88] update to v4 actions --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6da2dbd1..aec631bf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,7 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: | date sudo apt purge postgresql-14 @@ -48,7 +48,7 @@ jobs: - uses: denoland/setup-deno@v1 with: deno-version: v1.x - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version: ${{ matrix.node }} - run: npm test From 3e3d5e894a86b03b5e6edac9f52bd7ca4abd2ce5 Mon Sep 17 00:00:00 2001 From: "louis.tian" Date: Wed, 24 Jan 2024 11:02:11 +1100 Subject: [PATCH 42/88] add handler --- src/subscribe.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/subscribe.js b/src/subscribe.js index 7a70842e..decb42c6 100644 --- a/src/subscribe.js +++ b/src/subscribe.js @@ -47,7 +47,7 @@ export default function Subscribe(postgres, options) { return subscribe - async function subscribe(event, fn, onsubscribe = noop) { + async function subscribe(event, fn, onsubscribe = noop, onerror = noop) { event = parseEvent(event) if (!connection) @@ -66,6 +66,7 @@ export default function Subscribe(postgres, options) { return connection.then(x => { connected(x) onsubscribe() + stream && stream.on('error', onerror) return { unsubscribe, state, sql } }) } From 5404fbd6bcd145e604bc309b2e1a7cb49ceaed25 Mon Sep 17 00:00:00 2001 From: "louis.tian" Date: Thu, 25 Jan 2024 09:25:10 +1100 Subject: [PATCH 43/88] update type definition --- types/index.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/index.d.ts b/types/index.d.ts index 78d559ef..4e7b5653 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -690,7 +690,7 @@ declare namespace postgres { listen(channel: string, onnotify: (value: string) => void, onlisten?: (() => void) | undefined): ListenRequest; notify(channel: string, payload: string): PendingRequest; - subscribe(event: string, cb: (row: Row | null, info: ReplicationEvent) => void, onsubscribe?: (() => void) | undefined): Promise; + subscribe(event: string, cb: (row: Row | null, info: ReplicationEvent) => void, onsubscribe?: (() => void), onerror?: (() => any)): Promise; largeObject(oid?: number | undefined, /** @default 0x00020000 | 0x00040000 */ mode?: number | undefined): Promise; From a5cd8113cad622fafc1f6cfadccc11759ef36136 Mon Sep 17 00:00:00 2001 From: "louis.tian" Date: Mon, 29 Jan 2024 15:36:50 +1100 Subject: [PATCH 44/88] update lsn on Primary Keep Alive Message --- src/subscribe.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/subscribe.js b/src/subscribe.js index decb42c6..3db2f43b 100644 --- a/src/subscribe.js +++ b/src/subscribe.js @@ -110,8 +110,10 @@ export default function Subscribe(postgres, options) { function data(x) { if (x[0] === 0x77) parse(x.subarray(25), state, sql.options.parsers, handle, options.transform) - else if (x[0] === 0x6b && x[17]) + else if (x[0] === 0x6b && x[17]) { + state.lsn = x.subarray(1, 9) pong() + } } function handle(a, b) { From 9b6fc89d8705bf430bfc3e7f900450293fcdb8bb Mon Sep 17 00:00:00 2001 From: CDT Date: Tue, 13 Feb 2024 12:21:36 +0800 Subject: [PATCH 45/88] Update README.md fixed typo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c2202bcc..e6bf0ce8 100644 --- a/README.md +++ b/README.md @@ -270,7 +270,7 @@ const users = [ ] await sql` - update users set name = update_data.name, (age = update_data.age)::int + update users set name = update_data.name, age = (update_data.age)::int from (values ${sql(users)}) as update_data (id, name, age) where users.id = (update_data.id)::int returning users.id, users.name, users.age @@ -290,7 +290,7 @@ const users = await sql` or ```js -const [{ a, b, c }] => await sql` +const [{ a, b, c }] = await sql` select * from (values ${ sql(['a', 'b', 'c']) }) as x(a, b, c) From 4f648b3cfa5cb4bff7f0d0234929690f775e1801 Mon Sep 17 00:00:00 2001 From: Inklingboiii <69518450+Inklingboiii@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:28:06 +0100 Subject: [PATCH 46/88] Fixed Typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e6bf0ce8..1b93c156 100644 --- a/README.md +++ b/README.md @@ -917,7 +917,7 @@ The `Result` Array returned from queries is a custom array allowing for easy des ### .count -The `count` property is the number of affected rows returned by the database. This is usefull for insert, update and delete operations to know the number of rows since .length will be 0 in these cases if not using `RETURNING ...`. +The `count` property is the number of affected rows returned by the database. This is useful for insert, update and delete operations to know the number of rows since .length will be 0 in these cases if not using `RETURNING ...`. ### .command From 2b85ea7fb8b50f7c69232bd8074aa11c8cbe9d3a Mon Sep 17 00:00:00 2001 From: Ian Bytchek Date: Sat, 6 Jan 2024 10:28:31 +0000 Subject: [PATCH 47/88] Add `simple()` type definition Fixes #714. --- types/index.d.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/types/index.d.ts b/types/index.d.ts index 4e7b5653..8989ff47 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -599,6 +599,7 @@ declare namespace postgres { type RowList = T & Iterable> & ResultQueryMeta; interface PendingQueryModifiers { + simple(): this; readable(): Promise; writable(): Promise; From 3e28f3a596ccd1d309ac52972d6ef87a92bab26a Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Wed, 20 Mar 2024 10:32:00 +0100 Subject: [PATCH 48/88] Ensure retryRoutines are only used for prepared statements - fixes #830 --- src/connection.js | 2 +- tests/index.js | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/connection.js b/src/connection.js index a5694183..7f8ac5ea 100644 --- a/src/connection.js +++ b/src/connection.js @@ -788,7 +788,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose const error = Errors.postgres(parseError(x)) query && query.retried ? errored(query.retried) - : query && retryRoutines.has(error.routine) + : query && query.prepare && retryRoutines.has(error.routine) ? retry(query, error) : errored(error) } diff --git a/tests/index.js b/tests/index.js index cd08370a..13734239 100644 --- a/tests/index.js +++ b/tests/index.js @@ -1789,6 +1789,21 @@ t('Recreate prepared statements on RevalidateCachedQuery error', async() => { ] }) +t('Properly throws routing error on not prepared statements', async() => { + await sql`create table x (x text[])` + const { routine } = await sql.unsafe(`insert into x(x) values (('a', 'b'))`).catch(e => e) + + return ['transformAssignedExpr', routine, await sql`drop table x`] +}) + +t('Properly throws routing error on not prepared statements in transaction', async() => { + const { routine } = await sql.begin(sql => [ + sql`create table x (x text[])`, + sql`insert into x(x) values (('a', 'b'))`, + ]).catch(e => e) + + return ['transformAssignedExpr', routine] +}) t('Catches connection config errors', async() => { const sql = postgres({ ...options, user: { toString: () => { throw new Error('wat') } }, database: 'prut' }) From 6f20f3fe4510e25150e05306596f46e2688dc7f9 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Wed, 20 Mar 2024 19:28:27 +0100 Subject: [PATCH 49/88] build --- cjs/src/connection.js | 8 +++----- cjs/src/subscribe.js | 7 +++++-- cjs/tests/index.js | 15 +++++++++++++++ deno/README.md | 10 +++++----- deno/src/connection.js | 8 +++----- deno/src/subscribe.js | 7 +++++-- deno/tests/index.js | 15 +++++++++++++++ deno/types/index.d.ts | 3 ++- 8 files changed, 53 insertions(+), 20 deletions(-) diff --git a/cjs/src/connection.js b/cjs/src/connection.js index 425e91cd..9180693d 100644 --- a/cjs/src/connection.js +++ b/cjs/src/connection.js @@ -429,10 +429,8 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose lifeTimer.cancel() connectTimer.cancel() - if (socket.encrypted) { - socket.removeAllListeners() - socket = null - } + socket.removeAllListeners() + socket = null if (initial) return reconnect() @@ -790,7 +788,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose const error = Errors.postgres(parseError(x)) query && query.retried ? errored(query.retried) - : query && retryRoutines.has(error.routine) + : query && query.prepare && retryRoutines.has(error.routine) ? retry(query, error) : errored(error) } diff --git a/cjs/src/subscribe.js b/cjs/src/subscribe.js index 34d99e9f..e450071e 100644 --- a/cjs/src/subscribe.js +++ b/cjs/src/subscribe.js @@ -47,7 +47,7 @@ module.exports = Subscribe;function Subscribe(postgres, options) { return subscribe - async function subscribe(event, fn, onsubscribe = noop) { + async function subscribe(event, fn, onsubscribe = noop, onerror = noop) { event = parseEvent(event) if (!connection) @@ -66,6 +66,7 @@ module.exports = Subscribe;function Subscribe(postgres, options) { return connection.then(x => { connected(x) onsubscribe() + stream && stream.on('error', onerror) return { unsubscribe, state, sql } }) } @@ -109,8 +110,10 @@ module.exports = Subscribe;function Subscribe(postgres, options) { function data(x) { if (x[0] === 0x77) parse(x.subarray(25), state, sql.options.parsers, handle, options.transform) - else if (x[0] === 0x6b && x[17]) + else if (x[0] === 0x6b && x[17]) { + state.lsn = x.subarray(1, 9) pong() + } } function handle(a, b) { diff --git a/cjs/tests/index.js b/cjs/tests/index.js index 5aa0ae15..437ed2f9 100644 --- a/cjs/tests/index.js +++ b/cjs/tests/index.js @@ -1789,6 +1789,21 @@ t('Recreate prepared statements on RevalidateCachedQuery error', async() => { ] }) +t('Properly throws routing error on not prepared statements', async() => { + await sql`create table x (x text[])` + const { routine } = await sql.unsafe(`insert into x(x) values (('a', 'b'))`).catch(e => e) + + return ['transformAssignedExpr', routine, await sql`drop table x`] +}) + +t('Properly throws routing error on not prepared statements in transaction', async() => { + const { routine } = await sql.begin(sql => [ + sql`create table x (x text[])`, + sql`insert into x(x) values (('a', 'b'))`, + ]).catch(e => e) + + return ['transformAssignedExpr', routine] +}) t('Catches connection config errors', async() => { const sql = postgres({ ...options, user: { toString: () => { throw new Error('wat') } }, database: 'prut' }) diff --git a/deno/README.md b/deno/README.md index 0fc569bb..94a05714 100644 --- a/deno/README.md +++ b/deno/README.md @@ -266,7 +266,7 @@ const users = [ ] await sql` - update users set name = update_data.name, (age = update_data.age)::int + update users set name = update_data.name, age = (update_data.age)::int from (values ${sql(users)}) as update_data (id, name, age) where users.id = (update_data.id)::int returning users.id, users.name, users.age @@ -286,7 +286,7 @@ const users = await sql` or ```js -const [{ a, b, c }] => await sql` +const [{ a, b, c }] = await sql` select * from (values ${ sql(['a', 'b', 'c']) }) as x(a, b, c) @@ -913,7 +913,7 @@ The `Result` Array returned from queries is a custom array allowing for easy des ### .count -The `count` property is the number of affected rows returned by the database. This is usefull for insert, update and delete operations to know the number of rows since .length will be 0 in these cases if not using `RETURNING ...`. +The `count` property is the number of affected rows returned by the database. This is useful for insert, update and delete operations to know the number of rows since .length will be 0 in these cases if not using `RETURNING ...`. ### .command @@ -1099,10 +1099,10 @@ export default async fetch(req: Request, env: Env, ctx: ExecutionContext) { } ``` -In `wrangler.toml` you will need to enable `node_compat` to allow Postgres.js to operate in the Workers environment: +In `wrangler.toml` you will need to enable the `nodejs_compat` compatibility flag to allow Postgres.js to operate in the Workers environment: ```toml -node_compat = true # required for database drivers to function +compatibility_flags = ["nodejs_compat"] ``` ### Auto fetching of array types diff --git a/deno/src/connection.js b/deno/src/connection.js index 334b9722..2722095c 100644 --- a/deno/src/connection.js +++ b/deno/src/connection.js @@ -432,10 +432,8 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose lifeTimer.cancel() connectTimer.cancel() - if (socket.encrypted) { - socket.removeAllListeners() - socket = null - } + socket.removeAllListeners() + socket = null if (initial) return reconnect() @@ -793,7 +791,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose const error = Errors.postgres(parseError(x)) query && query.retried ? errored(query.retried) - : query && retryRoutines.has(error.routine) + : query && query.prepare && retryRoutines.has(error.routine) ? retry(query, error) : errored(error) } diff --git a/deno/src/subscribe.js b/deno/src/subscribe.js index dbb9b971..57316fa6 100644 --- a/deno/src/subscribe.js +++ b/deno/src/subscribe.js @@ -48,7 +48,7 @@ export default function Subscribe(postgres, options) { return subscribe - async function subscribe(event, fn, onsubscribe = noop) { + async function subscribe(event, fn, onsubscribe = noop, onerror = noop) { event = parseEvent(event) if (!connection) @@ -67,6 +67,7 @@ export default function Subscribe(postgres, options) { return connection.then(x => { connected(x) onsubscribe() + stream && stream.on('error', onerror) return { unsubscribe, state, sql } }) } @@ -110,8 +111,10 @@ export default function Subscribe(postgres, options) { function data(x) { if (x[0] === 0x77) parse(x.subarray(25), state, sql.options.parsers, handle, options.transform) - else if (x[0] === 0x6b && x[17]) + else if (x[0] === 0x6b && x[17]) { + state.lsn = x.subarray(1, 9) pong() + } } function handle(a, b) { diff --git a/deno/tests/index.js b/deno/tests/index.js index 90d1feeb..55581c42 100644 --- a/deno/tests/index.js +++ b/deno/tests/index.js @@ -1791,6 +1791,21 @@ t('Recreate prepared statements on RevalidateCachedQuery error', async() => { ] }) +t('Properly throws routing error on not prepared statements', async() => { + await sql`create table x (x text[])` + const { routine } = await sql.unsafe(`insert into x(x) values (('a', 'b'))`).catch(e => e) + + return ['transformAssignedExpr', routine, await sql`drop table x`] +}) + +t('Properly throws routing error on not prepared statements in transaction', async() => { + const { routine } = await sql.begin(sql => [ + sql`create table x (x text[])`, + sql`insert into x(x) values (('a', 'b'))`, + ]).catch(e => e) + + return ['transformAssignedExpr', routine] +}) t('Catches connection config errors', async() => { const sql = postgres({ ...options, user: { toString: () => { throw new Error('wat') } }, database: 'prut' }) diff --git a/deno/types/index.d.ts b/deno/types/index.d.ts index 6f96fe97..2088662d 100644 --- a/deno/types/index.d.ts +++ b/deno/types/index.d.ts @@ -601,6 +601,7 @@ declare namespace postgres { type RowList = T & Iterable> & ResultQueryMeta; interface PendingQueryModifiers { + simple(): this; readable(): Promise; writable(): Promise; @@ -692,7 +693,7 @@ declare namespace postgres { listen(channel: string, onnotify: (value: string) => void, onlisten?: (() => void) | undefined): ListenRequest; notify(channel: string, payload: string): PendingRequest; - subscribe(event: string, cb: (row: Row | null, info: ReplicationEvent) => void, onsubscribe?: (() => void) | undefined): Promise; + subscribe(event: string, cb: (row: Row | null, info: ReplicationEvent) => void, onsubscribe?: (() => void), onerror?: (() => any)): Promise; largeObject(oid?: number | undefined, /** @default 0x00020000 | 0x00040000 */ mode?: number | undefined): Promise; From f82ca1b85345650d5063745d80a61ac207826de1 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Wed, 20 Mar 2024 21:02:54 +0100 Subject: [PATCH 50/88] Properly check if prepared --- cjs/src/connection.js | 2 +- cjs/tests/index.js | 13 +++++++++++-- deno/src/connection.js | 2 +- deno/tests/index.js | 13 +++++++++++-- src/connection.js | 2 +- tests/index.js | 13 +++++++++++-- 6 files changed, 36 insertions(+), 9 deletions(-) diff --git a/cjs/src/connection.js b/cjs/src/connection.js index 9180693d..10184ca3 100644 --- a/cjs/src/connection.js +++ b/cjs/src/connection.js @@ -788,7 +788,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose const error = Errors.postgres(parseError(x)) query && query.retried ? errored(query.retried) - : query && query.prepare && retryRoutines.has(error.routine) + : query && query.prepared && retryRoutines.has(error.routine) ? retry(query, error) : errored(error) } diff --git a/cjs/tests/index.js b/cjs/tests/index.js index 437ed2f9..d49c7dcf 100644 --- a/cjs/tests/index.js +++ b/cjs/tests/index.js @@ -1789,14 +1789,14 @@ t('Recreate prepared statements on RevalidateCachedQuery error', async() => { ] }) -t('Properly throws routing error on not prepared statements', async() => { +t('Properly throws routine error on not prepared statements', async() => { await sql`create table x (x text[])` const { routine } = await sql.unsafe(`insert into x(x) values (('a', 'b'))`).catch(e => e) return ['transformAssignedExpr', routine, await sql`drop table x`] }) -t('Properly throws routing error on not prepared statements in transaction', async() => { +t('Properly throws routine error on not prepared statements in transaction', async() => { const { routine } = await sql.begin(sql => [ sql`create table x (x text[])`, sql`insert into x(x) values (('a', 'b'))`, @@ -1805,6 +1805,15 @@ t('Properly throws routing error on not prepared statements in transaction', asy return ['transformAssignedExpr', routine] }) +t('Properly throws routine error on not prepared statements using file', async() => { + const { routine } = await sql.unsafe(` + create table x (x text[]); + insert into x(x) values (('a', 'b')); + `, { prepare: true }).catch(e => e) + + return ['transformAssignedExpr', routine] +}) + t('Catches connection config errors', async() => { const sql = postgres({ ...options, user: { toString: () => { throw new Error('wat') } }, database: 'prut' }) diff --git a/deno/src/connection.js b/deno/src/connection.js index 2722095c..81f26c08 100644 --- a/deno/src/connection.js +++ b/deno/src/connection.js @@ -791,7 +791,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose const error = Errors.postgres(parseError(x)) query && query.retried ? errored(query.retried) - : query && query.prepare && retryRoutines.has(error.routine) + : query && query.prepared && retryRoutines.has(error.routine) ? retry(query, error) : errored(error) } diff --git a/deno/tests/index.js b/deno/tests/index.js index 55581c42..055f479b 100644 --- a/deno/tests/index.js +++ b/deno/tests/index.js @@ -1791,14 +1791,14 @@ t('Recreate prepared statements on RevalidateCachedQuery error', async() => { ] }) -t('Properly throws routing error on not prepared statements', async() => { +t('Properly throws routine error on not prepared statements', async() => { await sql`create table x (x text[])` const { routine } = await sql.unsafe(`insert into x(x) values (('a', 'b'))`).catch(e => e) return ['transformAssignedExpr', routine, await sql`drop table x`] }) -t('Properly throws routing error on not prepared statements in transaction', async() => { +t('Properly throws routine error on not prepared statements in transaction', async() => { const { routine } = await sql.begin(sql => [ sql`create table x (x text[])`, sql`insert into x(x) values (('a', 'b'))`, @@ -1807,6 +1807,15 @@ t('Properly throws routing error on not prepared statements in transaction', asy return ['transformAssignedExpr', routine] }) +t('Properly throws routine error on not prepared statements using file', async() => { + const { routine } = await sql.unsafe(` + create table x (x text[]); + insert into x(x) values (('a', 'b')); + `, { prepare: true }).catch(e => e) + + return ['transformAssignedExpr', routine] +}) + t('Catches connection config errors', async() => { const sql = postgres({ ...options, user: { toString: () => { throw new Error('wat') } }, database: 'prut' }) diff --git a/src/connection.js b/src/connection.js index 7f8ac5ea..578a6a02 100644 --- a/src/connection.js +++ b/src/connection.js @@ -788,7 +788,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose const error = Errors.postgres(parseError(x)) query && query.retried ? errored(query.retried) - : query && query.prepare && retryRoutines.has(error.routine) + : query && query.prepared && retryRoutines.has(error.routine) ? retry(query, error) : errored(error) } diff --git a/tests/index.js b/tests/index.js index 13734239..dd8d55da 100644 --- a/tests/index.js +++ b/tests/index.js @@ -1789,14 +1789,14 @@ t('Recreate prepared statements on RevalidateCachedQuery error', async() => { ] }) -t('Properly throws routing error on not prepared statements', async() => { +t('Properly throws routine error on not prepared statements', async() => { await sql`create table x (x text[])` const { routine } = await sql.unsafe(`insert into x(x) values (('a', 'b'))`).catch(e => e) return ['transformAssignedExpr', routine, await sql`drop table x`] }) -t('Properly throws routing error on not prepared statements in transaction', async() => { +t('Properly throws routine error on not prepared statements in transaction', async() => { const { routine } = await sql.begin(sql => [ sql`create table x (x text[])`, sql`insert into x(x) values (('a', 'b'))`, @@ -1805,6 +1805,15 @@ t('Properly throws routing error on not prepared statements in transaction', asy return ['transformAssignedExpr', routine] }) +t('Properly throws routine error on not prepared statements using file', async() => { + const { routine } = await sql.unsafe(` + create table x (x text[]); + insert into x(x) values (('a', 'b')); + `, { prepare: true }).catch(e => e) + + return ['transformAssignedExpr', routine] +}) + t('Catches connection config errors', async() => { const sql = postgres({ ...options, user: { toString: () => { throw new Error('wat') } }, database: 'prut' }) From 2524083cfbc39efc989f38dd4752ff08caa48bd1 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Thu, 21 Mar 2024 21:35:49 +0100 Subject: [PATCH 51/88] build --- cf/src/connection.js | 8 +++----- cf/src/subscribe.js | 7 +++++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cf/src/connection.js b/cf/src/connection.js index ab977ca8..c9231dc6 100644 --- a/cf/src/connection.js +++ b/cf/src/connection.js @@ -431,10 +431,8 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose lifeTimer.cancel() connectTimer.cancel() - if (socket.encrypted) { - socket.removeAllListeners() - socket = null - } + socket.removeAllListeners() + socket = null if (initial) return reconnect() @@ -792,7 +790,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose const error = Errors.postgres(parseError(x)) query && query.retried ? errored(query.retried) - : query && retryRoutines.has(error.routine) + : query && query.prepared && retryRoutines.has(error.routine) ? retry(query, error) : errored(error) } diff --git a/cf/src/subscribe.js b/cf/src/subscribe.js index 1ab8b0be..35a98d61 100644 --- a/cf/src/subscribe.js +++ b/cf/src/subscribe.js @@ -48,7 +48,7 @@ export default function Subscribe(postgres, options) { return subscribe - async function subscribe(event, fn, onsubscribe = noop) { + async function subscribe(event, fn, onsubscribe = noop, onerror = noop) { event = parseEvent(event) if (!connection) @@ -67,6 +67,7 @@ export default function Subscribe(postgres, options) { return connection.then(x => { connected(x) onsubscribe() + stream && stream.on('error', onerror) return { unsubscribe, state, sql } }) } @@ -110,8 +111,10 @@ export default function Subscribe(postgres, options) { function data(x) { if (x[0] === 0x77) parse(x.subarray(25), state, sql.options.parsers, handle, options.transform) - else if (x[0] === 0x6b && x[17]) + else if (x[0] === 0x6b && x[17]) { + state.lsn = x.subarray(1, 9) pong() + } } function handle(a, b) { From a42de3035848955b946b21ac108b164b6281f383 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Thu, 21 Mar 2024 21:48:18 +0100 Subject: [PATCH 52/88] please eslint --- cf/src/subscribe.js | 6 +++--- cjs/src/subscribe.js | 6 +++--- cjs/tests/index.js | 6 ++++-- deno/src/subscribe.js | 6 +++--- deno/tests/index.js | 6 ++++-- src/subscribe.js | 6 +++--- tests/index.js | 6 ++++-- 7 files changed, 24 insertions(+), 18 deletions(-) diff --git a/cf/src/subscribe.js b/cf/src/subscribe.js index 35a98d61..8716100e 100644 --- a/cf/src/subscribe.js +++ b/cf/src/subscribe.js @@ -105,13 +105,13 @@ export default function Subscribe(postgres, options) { return { stream, state: xs.state } function error(e) { - console.error('Unexpected error during logical streaming - reconnecting', e) + console.error('Unexpected error during logical streaming - reconnecting', e) // eslint-disable-line } function data(x) { - if (x[0] === 0x77) + if (x[0] === 0x77) { parse(x.subarray(25), state, sql.options.parsers, handle, options.transform) - else if (x[0] === 0x6b && x[17]) { + } else if (x[0] === 0x6b && x[17]) { state.lsn = x.subarray(1, 9) pong() } diff --git a/cjs/src/subscribe.js b/cjs/src/subscribe.js index e450071e..6aaa8962 100644 --- a/cjs/src/subscribe.js +++ b/cjs/src/subscribe.js @@ -104,13 +104,13 @@ module.exports = Subscribe;function Subscribe(postgres, options) { return { stream, state: xs.state } function error(e) { - console.error('Unexpected error during logical streaming - reconnecting', e) + console.error('Unexpected error during logical streaming - reconnecting', e) // eslint-disable-line } function data(x) { - if (x[0] === 0x77) + if (x[0] === 0x77) { parse(x.subarray(25), state, sql.options.parsers, handle, options.transform) - else if (x[0] === 0x6b && x[17]) { + } else if (x[0] === 0x6b && x[17]) { state.lsn = x.subarray(1, 9) pong() } diff --git a/cjs/tests/index.js b/cjs/tests/index.js index d49c7dcf..7d84ac67 100644 --- a/cjs/tests/index.js +++ b/cjs/tests/index.js @@ -1791,7 +1791,9 @@ t('Recreate prepared statements on RevalidateCachedQuery error', async() => { t('Properly throws routine error on not prepared statements', async() => { await sql`create table x (x text[])` - const { routine } = await sql.unsafe(`insert into x(x) values (('a', 'b'))`).catch(e => e) + const { routine } = await sql.unsafe(` + insert into x(x) values (('a', 'b')) + `).catch(e => e) return ['transformAssignedExpr', routine, await sql`drop table x`] }) @@ -1799,7 +1801,7 @@ t('Properly throws routine error on not prepared statements', async() => { t('Properly throws routine error on not prepared statements in transaction', async() => { const { routine } = await sql.begin(sql => [ sql`create table x (x text[])`, - sql`insert into x(x) values (('a', 'b'))`, + sql`insert into x(x) values (('a', 'b'))` ]).catch(e => e) return ['transformAssignedExpr', routine] diff --git a/deno/src/subscribe.js b/deno/src/subscribe.js index 57316fa6..b20efb96 100644 --- a/deno/src/subscribe.js +++ b/deno/src/subscribe.js @@ -105,13 +105,13 @@ export default function Subscribe(postgres, options) { return { stream, state: xs.state } function error(e) { - console.error('Unexpected error during logical streaming - reconnecting', e) + console.error('Unexpected error during logical streaming - reconnecting', e) // eslint-disable-line } function data(x) { - if (x[0] === 0x77) + if (x[0] === 0x77) { parse(x.subarray(25), state, sql.options.parsers, handle, options.transform) - else if (x[0] === 0x6b && x[17]) { + } else if (x[0] === 0x6b && x[17]) { state.lsn = x.subarray(1, 9) pong() } diff --git a/deno/tests/index.js b/deno/tests/index.js index 055f479b..754eabd3 100644 --- a/deno/tests/index.js +++ b/deno/tests/index.js @@ -1793,7 +1793,9 @@ t('Recreate prepared statements on RevalidateCachedQuery error', async() => { t('Properly throws routine error on not prepared statements', async() => { await sql`create table x (x text[])` - const { routine } = await sql.unsafe(`insert into x(x) values (('a', 'b'))`).catch(e => e) + const { routine } = await sql.unsafe(` + insert into x(x) values (('a', 'b')) + `).catch(e => e) return ['transformAssignedExpr', routine, await sql`drop table x`] }) @@ -1801,7 +1803,7 @@ t('Properly throws routine error on not prepared statements', async() => { t('Properly throws routine error on not prepared statements in transaction', async() => { const { routine } = await sql.begin(sql => [ sql`create table x (x text[])`, - sql`insert into x(x) values (('a', 'b'))`, + sql`insert into x(x) values (('a', 'b'))` ]).catch(e => e) return ['transformAssignedExpr', routine] diff --git a/src/subscribe.js b/src/subscribe.js index 3db2f43b..4f8934cc 100644 --- a/src/subscribe.js +++ b/src/subscribe.js @@ -104,13 +104,13 @@ export default function Subscribe(postgres, options) { return { stream, state: xs.state } function error(e) { - console.error('Unexpected error during logical streaming - reconnecting', e) + console.error('Unexpected error during logical streaming - reconnecting', e) // eslint-disable-line } function data(x) { - if (x[0] === 0x77) + if (x[0] === 0x77) { parse(x.subarray(25), state, sql.options.parsers, handle, options.transform) - else if (x[0] === 0x6b && x[17]) { + } else if (x[0] === 0x6b && x[17]) { state.lsn = x.subarray(1, 9) pong() } diff --git a/tests/index.js b/tests/index.js index dd8d55da..bf81b036 100644 --- a/tests/index.js +++ b/tests/index.js @@ -1791,7 +1791,9 @@ t('Recreate prepared statements on RevalidateCachedQuery error', async() => { t('Properly throws routine error on not prepared statements', async() => { await sql`create table x (x text[])` - const { routine } = await sql.unsafe(`insert into x(x) values (('a', 'b'))`).catch(e => e) + const { routine } = await sql.unsafe(` + insert into x(x) values (('a', 'b')) + `).catch(e => e) return ['transformAssignedExpr', routine, await sql`drop table x`] }) @@ -1799,7 +1801,7 @@ t('Properly throws routine error on not prepared statements', async() => { t('Properly throws routine error on not prepared statements in transaction', async() => { const { routine } = await sql.begin(sql => [ sql`create table x (x text[])`, - sql`insert into x(x) values (('a', 'b'))`, + sql`insert into x(x) values (('a', 'b'))` ]).catch(e => e) return ['transformAssignedExpr', routine] From 3eb40995fe8d878b40a69ce75fedf55f7c298ce0 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Thu, 21 Mar 2024 21:51:08 +0100 Subject: [PATCH 53/88] 3.4.4 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ea500a80..4fb9a160 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "postgres", - "version": "3.4.3", + "version": "3.4.4", "description": "Fastest full featured PostgreSQL client for Node.js", "type": "module", "module": "src/index.js", From b8fa8f465429bbc9f9d64894f7b7769bc92762eb Mon Sep 17 00:00:00 2001 From: Heb Date: Thu, 29 Feb 2024 01:05:31 +0700 Subject: [PATCH 54/88] chore: update export conditions Hello there ! The official runtime export key for cloudflare is `workerd` (not worker). I believe many apps out there might already be relying on `worker` so I propose to add it alongside it. Reference : - https://developers.cloudflare.com/workers/wrangler/bundling/#conditional-exports - https://runtime-keys.proposal.wintercg.org/#workerd --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 4fb9a160..cd1545b9 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "types": "./types/index.d.ts", "bun": "./src/index.js", "worker": "./cf/src/index.js", + "workerd": "./cf/src/index.js", "import": "./src/index.js", "default": "./cjs/src/index.js" }, From cc688c642fc98c4338523d3e281e03bf0c3417b8 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 9 Apr 2024 22:22:21 +0200 Subject: [PATCH 55/88] Remove "worker" export as we now have "workerd" for cloudflare --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index cd1545b9..47f3add2 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,6 @@ "exports": { "types": "./types/index.d.ts", "bun": "./src/index.js", - "worker": "./cf/src/index.js", "workerd": "./cf/src/index.js", "import": "./src/index.js", "default": "./cjs/src/index.js" From 6bed5c0975ad78400b5b3f09767b3ea908d3b808 Mon Sep 17 00:00:00 2001 From: oakgary <13177748+oakgary@users.noreply.github.com> Date: Wed, 15 May 2024 12:26:36 +0200 Subject: [PATCH 56/88] corrects explnation of default max_lifetime values in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b93c156..421d19a0 100644 --- a/README.md +++ b/README.md @@ -992,7 +992,7 @@ const sql = postgres('postgres://username:password@host:port/database', { }) ``` -Note that `max_lifetime = 60 * (30 + Math.random() * 30)` by default. This resolves to an interval between 45 and 90 minutes to optimize for the benefits of prepared statements **and** working nicely with Linux's OOM killer. +Note that `max_lifetime = 60 * (30 + Math.random() * 30)` by default. This resolves to an interval between 30 and 60 minutes to optimize for the benefits of prepared statements **and** working nicely with Linux's OOM killer. ### Dynamic passwords From e05585bdbd020640a7ae19e08ff78b9aa66e1c66 Mon Sep 17 00:00:00 2001 From: oakgary <13177748+oakgary@users.noreply.github.com> Date: Wed, 15 May 2024 12:33:15 +0200 Subject: [PATCH 57/88] corrects explnation of default max_lifetime values in deno/README.md --- deno/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deno/README.md b/deno/README.md index 94a05714..31ea4aea 100644 --- a/deno/README.md +++ b/deno/README.md @@ -988,7 +988,7 @@ const sql = postgres('postgres://username:password@host:port/database', { }) ``` -Note that `max_lifetime = 60 * (30 + Math.random() * 30)` by default. This resolves to an interval between 45 and 90 minutes to optimize for the benefits of prepared statements **and** working nicely with Linux's OOM killer. +Note that `max_lifetime = 60 * (30 + Math.random() * 30)` by default. This resolves to an interval between 30 and 60 minutes to optimize for the benefits of prepared statements **and** working nicely with Linux's OOM killer. ### Dynamic passwords From f58cd4f3affd3e8ce8f53e42799672d86cd2c70b Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 30 Apr 2024 14:03:04 +0200 Subject: [PATCH 58/88] Don't reassign to errors --- cjs/src/connection.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cjs/src/connection.js b/cjs/src/connection.js index 10184ca3..3b913a47 100644 --- a/cjs/src/connection.js +++ b/cjs/src/connection.js @@ -385,7 +385,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } function queryError(query, err) { - Object.defineProperties(err, { + 'parameters' in err || Object.defineProperties(err, { stack: { value: err.stack + query.origin.replace(/.*\n/, '\n'), enumerable: options.debug }, query: { value: query.string, enumerable: options.debug }, parameters: { value: query.parameters, enumerable: options.debug }, From 4baef5e4c6fbf6e655da033bfde2a7193623329a Mon Sep 17 00:00:00 2001 From: "Ch3rry B@ry" Date: Thu, 8 Aug 2024 12:32:03 +0530 Subject: [PATCH 59/88] Don't reassign to errors --- src/connection.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connection.js b/src/connection.js index 578a6a02..97cc97e1 100644 --- a/src/connection.js +++ b/src/connection.js @@ -385,7 +385,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } function queryError(query, err) { - Object.defineProperties(err, { + 'query' in err || 'parameters' in err || Object.defineProperties(err, { stack: { value: err.stack + query.origin.replace(/.*\n/, '\n'), enumerable: options.debug }, query: { value: query.string, enumerable: options.debug }, parameters: { value: query.parameters, enumerable: options.debug }, From 18186998b89a8ec60b82fd7140783f8833810e2d Mon Sep 17 00:00:00 2001 From: Andrew Harvey Date: Wed, 18 Sep 2024 10:15:01 +1000 Subject: [PATCH 60/88] Update README.md to fix broken link to Node.js stream backpressure documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 421d19a0..7c7b83f0 100644 --- a/README.md +++ b/README.md @@ -537,7 +537,7 @@ for await (const chunk of readableStream) { } ``` -> **NOTE** This is a low-level API which does not provide any type safety. To make this work, you must match your [`copy query` parameters](https://www.postgresql.org/docs/14/sql-copy.html) correctly to your [Node.js stream read or write](https://nodejs.org/api/stream.html) code. Ensure [Node.js stream backpressure](https://nodejs.org/en/docs/guides/backpressuring-in-streams/) is handled correctly to avoid memory exhaustion. +> **NOTE** This is a low-level API which does not provide any type safety. To make this work, you must match your [`copy query` parameters](https://www.postgresql.org/docs/14/sql-copy.html) correctly to your [Node.js stream read or write](https://nodejs.org/api/stream.html) code. Ensure [Node.js stream backpressure](https://nodejs.org/en/learn/modules/backpressuring-in-streams) is handled correctly to avoid memory exhaustion. ### Canceling Queries in Progress From 75dab3771074cec8595c0a403d1e19218017415c Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Fri, 27 Sep 2024 12:30:54 +0200 Subject: [PATCH 61/88] Try postgres 17 (might be too soon) --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aec631bf..48948290 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ jobs: fail-fast: false matrix: node: ['12', '14', '16', '18', '20', '21'] - postgres: ['12', '13', '14', '15', '16'] + postgres: ['12', '13', '14', '15', '16', '17'] runs-on: ubuntu-latest services: postgres: From f84f21a282b7a15ccb5cba6bb772f815bf0467f5 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Fri, 27 Sep 2024 12:42:12 +0200 Subject: [PATCH 62/88] also node 22 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 48948290..bf65797a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - node: ['12', '14', '16', '18', '20', '21'] + node: ['12', '14', '16', '18', '20', '21', '22'] postgres: ['12', '13', '14', '15', '16', '17'] runs-on: ubuntu-latest services: From 5fb70c14c08c7f378562571ea66ee7a69f19bd17 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Fri, 25 Oct 2024 10:36:31 +0200 Subject: [PATCH 63/88] Fix for deno 2 --- transpile.deno.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transpile.deno.js b/transpile.deno.js index 923ac9af..f077677b 100644 --- a/transpile.deno.js +++ b/transpile.deno.js @@ -55,7 +55,7 @@ function transpile(x, name, folder) { .replace('{ spawnSync }', '{ spawn }') } if (name === 'index.js') - x += '\n;window.addEventListener("unload", () => Deno.exit(process.exitCode))' + x += '\n;globalThis.addEventListener("unload", () => Deno.exit(process.exitCode))' } const buffer = x.includes('Buffer') From 5974e7fcc171e456737e9eb34a90f0ba2ea6ef56 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Fri, 25 Oct 2024 10:36:45 +0200 Subject: [PATCH 64/88] build --- cf/src/connection.js | 2 +- cjs/src/connection.js | 2 +- deno/README.md | 2 +- deno/src/connection.js | 2 +- deno/tests/index.js | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cf/src/connection.js b/cf/src/connection.js index c9231dc6..ee8b1e69 100644 --- a/cf/src/connection.js +++ b/cf/src/connection.js @@ -387,7 +387,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } function queryError(query, err) { - Object.defineProperties(err, { + 'query' in err || 'parameters' in err || Object.defineProperties(err, { stack: { value: err.stack + query.origin.replace(/.*\n/, '\n'), enumerable: options.debug }, query: { value: query.string, enumerable: options.debug }, parameters: { value: query.parameters, enumerable: options.debug }, diff --git a/cjs/src/connection.js b/cjs/src/connection.js index 3b913a47..f7f58d14 100644 --- a/cjs/src/connection.js +++ b/cjs/src/connection.js @@ -385,7 +385,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } function queryError(query, err) { - 'parameters' in err || Object.defineProperties(err, { + 'query' in err || 'parameters' in err || Object.defineProperties(err, { stack: { value: err.stack + query.origin.replace(/.*\n/, '\n'), enumerable: options.debug }, query: { value: query.string, enumerable: options.debug }, parameters: { value: query.parameters, enumerable: options.debug }, diff --git a/deno/README.md b/deno/README.md index 31ea4aea..6f8085cf 100644 --- a/deno/README.md +++ b/deno/README.md @@ -533,7 +533,7 @@ for await (const chunk of readableStream) { } ``` -> **NOTE** This is a low-level API which does not provide any type safety. To make this work, you must match your [`copy query` parameters](https://www.postgresql.org/docs/14/sql-copy.html) correctly to your [Node.js stream read or write](https://nodejs.org/api/stream.html) code. Ensure [Node.js stream backpressure](https://nodejs.org/en/docs/guides/backpressuring-in-streams/) is handled correctly to avoid memory exhaustion. +> **NOTE** This is a low-level API which does not provide any type safety. To make this work, you must match your [`copy query` parameters](https://www.postgresql.org/docs/14/sql-copy.html) correctly to your [Node.js stream read or write](https://nodejs.org/api/stream.html) code. Ensure [Node.js stream backpressure](https://nodejs.org/en/learn/modules/backpressuring-in-streams) is handled correctly to avoid memory exhaustion. ### Canceling Queries in Progress diff --git a/deno/src/connection.js b/deno/src/connection.js index 81f26c08..1726a9aa 100644 --- a/deno/src/connection.js +++ b/deno/src/connection.js @@ -388,7 +388,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } function queryError(query, err) { - Object.defineProperties(err, { + 'query' in err || 'parameters' in err || Object.defineProperties(err, { stack: { value: err.stack + query.origin.replace(/.*\n/, '\n'), enumerable: options.debug }, query: { value: query.string, enumerable: options.debug }, parameters: { value: query.parameters, enumerable: options.debug }, diff --git a/deno/tests/index.js b/deno/tests/index.js index 754eabd3..5b5d6e57 100644 --- a/deno/tests/index.js +++ b/deno/tests/index.js @@ -2583,4 +2583,4 @@ t('arrays in reserved connection', async() => { ] }) -;window.addEventListener("unload", () => Deno.exit(process.exitCode)) \ No newline at end of file +;globalThis.addEventListener("unload", () => Deno.exit(process.exitCode)) \ No newline at end of file From b231b688489212e40ab54e9870f84f55f2be5dd0 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Fri, 25 Oct 2024 10:42:52 +0200 Subject: [PATCH 65/88] 3.4.5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 47f3add2..d53fe2ca 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "postgres", - "version": "3.4.4", + "version": "3.4.5", "description": "Fastest full featured PostgreSQL client for Node.js", "type": "module", "module": "src/index.js", From 9f38ea1c2e2ab88c4b1c207c32c68ee47c327e2a Mon Sep 17 00:00:00 2001 From: gimse <23360355+gimse@users.noreply.github.com> Date: Sun, 7 Jul 2024 10:07:18 +0200 Subject: [PATCH 66/88] adding env.PGUSERNAME || --- src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index 0573e2bc..b6a9a9f7 100644 --- a/src/index.js +++ b/src/index.js @@ -480,7 +480,7 @@ function parseOptions(a, b) { {} ), connection : { - application_name: 'postgres.js', + application_name: env.PGUSERNAME || 'postgres.js', ...o.connection, ...Object.entries(query).reduce((acc, [k, v]) => (k in defaults || (acc[k] = v), acc), {}) }, From be716e220066470436012d76eec850a37de2f077 Mon Sep 17 00:00:00 2001 From: gimse <23360355+gimse@users.noreply.github.com> Date: Sun, 7 Jul 2024 10:14:15 +0200 Subject: [PATCH 67/88] README --- README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7c7b83f0..8f59cef4 100644 --- a/README.md +++ b/README.md @@ -1125,15 +1125,16 @@ It is also possible to connect to the database without a connection string or an const sql = postgres() ``` -| Option | Environment Variables | -| ----------------- | ------------------------ | -| `host` | `PGHOST` | -| `port` | `PGPORT` | -| `database` | `PGDATABASE` | -| `username` | `PGUSERNAME` or `PGUSER` | -| `password` | `PGPASSWORD` | -| `idle_timeout` | `PGIDLE_TIMEOUT` | -| `connect_timeout` | `PGCONNECT_TIMEOUT` | +| Option | Environment Variables | +| ------------------ | ------------------------ | +| `host` | `PGHOST` | +| `port` | `PGPORT` | +| `database` | `PGDATABASE` | +| `username` | `PGUSERNAME` or `PGUSER` | +| `password` | `PGPASSWORD` | +| `application_name` | `PGAPPNAME` | +| `idle_timeout` | `PGIDLE_TIMEOUT` | +| `connect_timeout` | `PGCONNECT_TIMEOUT` | ### Prepared statements From ef7afdb817d00cc7208bd1cefa88f861bfc2cbde Mon Sep 17 00:00:00 2001 From: gimse <23360355+gimse@users.noreply.github.com> Date: Sun, 7 Jul 2024 10:14:47 +0200 Subject: [PATCH 68/88] env.PGAPPNAME || --- src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index b6a9a9f7..2dfd24e8 100644 --- a/src/index.js +++ b/src/index.js @@ -480,7 +480,7 @@ function parseOptions(a, b) { {} ), connection : { - application_name: env.PGUSERNAME || 'postgres.js', + application_name: env.PGAPPNAME || 'postgres.js', ...o.connection, ...Object.entries(query).reduce((acc, [k, v]) => (k in defaults || (acc[k] = v), acc), {}) }, From 4099f3412bb1d9f58ef223e7e4444bc5e4a74a2d Mon Sep 17 00:00:00 2001 From: gimse <23360355+gimse@users.noreply.github.com> Date: Sun, 7 Jul 2024 10:35:18 +0200 Subject: [PATCH 69/88] changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8939f7c8..ed7ec4f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## v3.3.0 - 9 July 2024 +- Adding support for the PGAPPNAME environment variable + ## v3.2.4 - 25 May 2022 - Allow setting keep_alive: false bee62f3 - Fix support for null in arrays - fixes #371 b04c853 From a2c7de12b3bfc6809051d94ba6115150f80678e3 Mon Sep 17 00:00:00 2001 From: gimse <23360355+gimse@users.noreply.github.com> Date: Sun, 7 Jul 2024 15:01:06 +0200 Subject: [PATCH 70/88] removing change log --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed7ec4f8..8939f7c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,5 @@ # Changelog -## v3.3.0 - 9 July 2024 -- Adding support for the PGAPPNAME environment variable - ## v3.2.4 - 25 May 2022 - Allow setting keep_alive: false bee62f3 - Fix support for null in arrays - fixes #371 b04c853 From 6ec85a432b17661ccacbdf7f765c651e88969d36 Mon Sep 17 00:00:00 2001 From: mrl5 <31549762+mrl5@users.noreply.github.com> Date: Fri, 24 May 2024 19:34:59 +0200 Subject: [PATCH 71/88] docs(readme): mention pgbouncer supports protocol-level named prepared statements --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 8f59cef4..1dcdd668 100644 --- a/README.md +++ b/README.md @@ -1140,6 +1140,10 @@ const sql = postgres() Prepared statements will automatically be created for any queries where it can be inferred that the query is static. This can be disabled by using the `prepare: false` option. For instance — this is useful when [using PGBouncer in `transaction mode`](https://github.com/porsager/postgres/issues/93#issuecomment-656290493). +**update**: [since 1.21.0](https://www.pgbouncer.org/2023/10/pgbouncer-1-21-0) +PGBouncer supports protocol-level named prepared statements when [configured +properly](https://www.pgbouncer.org/config.html#max_prepared_statements) + ## Custom Types You can add ergonomic support for custom types, or simply use `sql.typed(value, type)` inline, where type is the PostgreSQL `oid` for the type and the correctly serialized string. _(`oid` values for types can be found in the `pg_catalog.pg_type` table.)_ From 93f5686ff9cd86ab0590e79b4d94f984a40183ad Mon Sep 17 00:00:00 2001 From: Valentinas Janeiko Date: Tue, 21 Jan 2025 19:51:25 +0000 Subject: [PATCH 72/88] chore: fix CI --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bf65797a..8ae323dd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,6 @@ jobs: - uses: actions/checkout@v4 - run: | date - sudo apt purge postgresql-14 sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo apt-get update From 3374a8aeb681b9d573459f1f5897c854a367cc55 Mon Sep 17 00:00:00 2001 From: Valentinas Janeiko <2305836+valeneiko@users.noreply.github.com> Date: Tue, 21 Jan 2025 20:29:12 +0000 Subject: [PATCH 73/88] try purging PG16 instead? --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8ae323dd..af00f7e0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,6 +28,7 @@ jobs: - uses: actions/checkout@v4 - run: | date + sudo apt purge postgresql-16 sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo apt-get update From 089214e85c23c90cf142d47fb30bd03f42874984 Mon Sep 17 00:00:00 2001 From: Louis Orleans Date: Tue, 21 Jan 2025 17:16:15 -0800 Subject: [PATCH 74/88] =?UTF-8?q?=E2=9C=8F=EF=B8=8F=20fix=20CONNECT=5FTIME?= =?UTF-8?q?OUT=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `CONNECT_TIMEOUT` error's name is `CONNECT_TIMEOUT`, not `CONNECTION_CONNECT_TIMEOUT`. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1dcdd668..b79c207f 100644 --- a/README.md +++ b/README.md @@ -1303,8 +1303,8 @@ This error is thrown if the user has called [`sql.end()`](#teardown--cleanup) an This error is thrown for any queries that were pending when the timeout to [`sql.end({ timeout: X })`](#teardown--cleanup) was reached. -##### CONNECTION_CONNECT_TIMEOUT -> write CONNECTION_CONNECT_TIMEOUT host:port +##### CONNECT_TIMEOUT +> write CONNECT_TIMEOUT host:port This error is thrown if the startup phase of the connection (tcp, protocol negotiation, and auth) took more than the default 30 seconds or what was specified using `connect_timeout` or `PGCONNECT_TIMEOUT`. From ad0ed4476e09f41f147859cb5a42971d2b99e9c7 Mon Sep 17 00:00:00 2001 From: adrtivv Date: Fri, 11 Apr 2025 00:37:07 +0530 Subject: [PATCH 75/88] fixed typings for generic error code variants --- types/index.d.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/types/index.d.ts b/types/index.d.ts index 8989ff47..eb604918 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -456,7 +456,8 @@ declare namespace postgres { | 'NOT_TAGGED_CALL' | 'UNDEFINED_VALUE' | 'MAX_PARAMETERS_EXCEEDED' - | 'SASL_SIGNATURE_MISMATCH'; + | 'SASL_SIGNATURE_MISMATCH' + | 'UNSAFE_TRANSACTION'; message: string; } From b0d8c8f363e006a74472d76f859da60c52a80368 Mon Sep 17 00:00:00 2001 From: Stephen Haberman Date: Sat, 3 May 2025 21:23:22 -0500 Subject: [PATCH 76/88] docs: Add prepare: true to sql.unsafe docs. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b79c207f..c135cd17 100644 --- a/README.md +++ b/README.md @@ -568,6 +568,8 @@ If you know what you're doing, you can use `unsafe` to pass any string you'd lik sql.unsafe('select ' + danger + ' from users where id = ' + dragons) ``` +By default, `sql.unsafe` assumes the `query` string is sufficiently dynamic that prepared statements do not make sense, and so defaults them to off. If you'd like to re-enable prepared statements, you can pass `{ prepare: true }`. + You can also nest `sql.unsafe` within a safe `sql` expression. This is useful if only part of your fraction has unsafe elements. ```js From 26ee4782b456207d48571c3469c50ada013b06cc Mon Sep 17 00:00:00 2001 From: madflow Date: Sun, 23 Feb 2025 17:46:44 +0100 Subject: [PATCH 77/88] docs: dynamic ordering --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index c135cd17..b04ac21c 100644 --- a/README.md +++ b/README.md @@ -342,6 +342,27 @@ select * from users select * from users where user_id = $1 ``` +### Dynamic ordering + +```js +const id = 1 +const order = { + username: 'asc' + created_at: 'desc' +} +await sql` + select + * + from ticket + where account = ${ id } + order by ${ + Object.entries(order).flatMap(([column, order], i) => + [i ? sql`,` : sql``, sql`${ sql(column) } ${ order === 'desc' ? sql`desc` : sql`asc` }`] + ) + } +` +``` + ### SQL functions Using keywords or calling functions dynamically is also possible by using ``` sql`` ``` fragments. ```js From a39dfefd507c3ea5568377faef531a8f6e0d90b6 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 20 May 2025 16:08:13 -0600 Subject: [PATCH 78/88] Ensure non object errors thrown are handled properly --- src/connection.js | 3 +++ tests/index.js | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/connection.js b/src/connection.js index 97cc97e1..44af2bee 100644 --- a/src/connection.js +++ b/src/connection.js @@ -385,6 +385,9 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } function queryError(query, err) { + if (!err || typeof err !== 'object') + err = new Error(err) + 'query' in err || 'parameters' in err || Object.defineProperties(err, { stack: { value: err.stack + query.origin.replace(/.*\n/, '\n'), enumerable: options.debug }, query: { value: query.string, enumerable: options.debug }, diff --git a/tests/index.js b/tests/index.js index bf81b036..65635399 100644 --- a/tests/index.js +++ b/tests/index.js @@ -429,6 +429,22 @@ t('Reconnect using SSL', { timeout: 2 }, async() => { return [1, (await sql`select 1 as x`)[0].x] }) +t('Proper handling of non object Errors', async() => { + const sql = postgres({ socket: () => { throw 'wat' } }) + + return [ + 'wat', await sql`select 1 as x`.catch(e => e.message) + ] +}) + +t('Proper handling of null Errors', async() => { + const sql = postgres({ socket: () => { throw null } }) + + return [ + 'null', await sql`select 1 as x`.catch(e => e.message) + ] +}) + t('Login without password', async() => { return [true, (await postgres({ ...options, ...login })`select true as x`)[0].x] }) From 76c13f2f2f5a57f52bf927c9942912b70ce8a65c Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 20 May 2025 16:58:09 -0600 Subject: [PATCH 79/88] Fix stuck queries on non busy connections --- src/connection.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connection.js b/src/connection.js index 44af2bee..8e71751a 100644 --- a/src/connection.js +++ b/src/connection.js @@ -293,7 +293,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose if (incomings) { incomings.push(x) remaining -= x.length - if (remaining >= 0) + if (remaining > 0) return } From e8bb3b829ae2058d70694b397de434d046091a34 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 20 May 2025 16:59:07 -0600 Subject: [PATCH 80/88] Fix failures in sql.reserve() connect shadowing real errors fixes #778 #923 #944 #1028 --- src/connection.js | 2 +- src/index.js | 7 ++++--- tests/index.js | 8 ++++++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/connection.js b/src/connection.js index 8e71751a..b1ff1244 100644 --- a/src/connection.js +++ b/src/connection.js @@ -381,7 +381,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose function errored(err) { stream && (stream.destroy(err), stream = null) query && queryError(query, err) - initial && (queryError(initial, err), initial = null) + initial && (initial.reserve ? initial.reject(err) : queryError(initial, err), initial = null) } function queryError(query, err) { diff --git a/src/index.js b/src/index.js index 2dfd24e8..944d50cf 100644 --- a/src/index.js +++ b/src/index.js @@ -204,9 +204,10 @@ function Postgres(a, b) { const queue = Queue() const c = open.length ? open.shift() - : await new Promise(r => { - queries.push({ reserve: r }) - closed.length && connect(closed.shift()) + : await new Promise((resolve, reject) => { + const query = { reserve: resolve, reject } + queries.push(query) + closed.length && connect(closed.shift(), query) }) move(c, reserved) diff --git a/tests/index.js b/tests/index.js index 65635399..5b5e6f87 100644 --- a/tests/index.js +++ b/tests/index.js @@ -445,6 +445,14 @@ t('Proper handling of null Errors', async() => { ] }) +t('Ensure reserve throws proper error', async() => { + const sql = postgres({ socket: () => { throw 'wat' }, idle_timeout }) + + return [ + 'wat', await sql.reserve().catch(e => e) + ] +}) + t('Login without password', async() => { return [true, (await postgres({ ...options, ...login })`select true as x`)[0].x] }) From 36dbe2fbf500dac8cea979b620ccc1e2e10d0de5 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 20 May 2025 17:09:46 -0600 Subject: [PATCH 81/88] --unstable not needed for deno, but use --no-lock to not include deno.lock --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d53fe2ca..f556cd28 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "test": "npm run test:esm && npm run test:cjs && npm run test:deno", "test:esm": "node tests/index.js", "test:cjs": "npm run build:cjs && cd cjs/tests && node index.js && cd ../../", - "test:deno": "npm run build:deno && cd deno/tests && deno run --unstable --allow-all --unsafely-ignore-certificate-errors index.js && cd ../../", + "test:deno": "npm run build:deno && cd deno/tests && deno run --no-lock --allow-all --unsafely-ignore-certificate-errors index.js && cd ../../", "lint": "eslint src && eslint tests", "prepare": "npm run build", "prepublishOnly": "npm run lint" From a92f4705968cebc0451d6ab3270b26a48a112318 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 20 May 2025 17:20:56 -0600 Subject: [PATCH 82/88] build --- cf/src/connection.js | 7 ++++-- cf/src/index.js | 9 ++++---- cjs/src/connection.js | 7 ++++-- cjs/src/index.js | 9 ++++---- cjs/tests/index.js | 24 ++++++++++++++++++++ deno/README.md | 50 ++++++++++++++++++++++++++++++++---------- deno/src/connection.js | 7 ++++-- deno/src/index.js | 9 ++++---- deno/tests/index.js | 24 ++++++++++++++++++++ deno/types/index.d.ts | 3 ++- tests/index.js | 6 ++--- 11 files changed, 122 insertions(+), 33 deletions(-) diff --git a/cf/src/connection.js b/cf/src/connection.js index ee8b1e69..d918b136 100644 --- a/cf/src/connection.js +++ b/cf/src/connection.js @@ -295,7 +295,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose if (incomings) { incomings.push(x) remaining -= x.length - if (remaining >= 0) + if (remaining > 0) return } @@ -383,10 +383,13 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose function errored(err) { stream && (stream.destroy(err), stream = null) query && queryError(query, err) - initial && (queryError(initial, err), initial = null) + initial && (initial.reserve ? initial.reject(err) : queryError(initial, err), initial = null) } function queryError(query, err) { + if (!err || typeof err !== 'object') + err = new Error(err) + 'query' in err || 'parameters' in err || Object.defineProperties(err, { stack: { value: err.stack + query.origin.replace(/.*\n/, '\n'), enumerable: options.debug }, query: { value: query.string, enumerable: options.debug }, diff --git a/cf/src/index.js b/cf/src/index.js index d24e9f9c..3ffb7e65 100644 --- a/cf/src/index.js +++ b/cf/src/index.js @@ -205,9 +205,10 @@ function Postgres(a, b) { const queue = Queue() const c = open.length ? open.shift() - : await new Promise(r => { - queries.push({ reserve: r }) - closed.length && connect(closed.shift()) + : await new Promise((resolve, reject) => { + const query = { reserve: resolve, reject } + queries.push(query) + closed.length && connect(closed.shift(), query) }) move(c, reserved) @@ -481,7 +482,7 @@ function parseOptions(a, b) { {} ), connection : { - application_name: 'postgres.js', + application_name: env.PGAPPNAME || 'postgres.js', ...o.connection, ...Object.entries(query).reduce((acc, [k, v]) => (k in defaults || (acc[k] = v), acc), {}) }, diff --git a/cjs/src/connection.js b/cjs/src/connection.js index f7f58d14..db427247 100644 --- a/cjs/src/connection.js +++ b/cjs/src/connection.js @@ -293,7 +293,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose if (incomings) { incomings.push(x) remaining -= x.length - if (remaining >= 0) + if (remaining > 0) return } @@ -381,10 +381,13 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose function errored(err) { stream && (stream.destroy(err), stream = null) query && queryError(query, err) - initial && (queryError(initial, err), initial = null) + initial && (initial.reserve ? initial.reject(err) : queryError(initial, err), initial = null) } function queryError(query, err) { + if (!err || typeof err !== 'object') + err = new Error(err) + 'query' in err || 'parameters' in err || Object.defineProperties(err, { stack: { value: err.stack + query.origin.replace(/.*\n/, '\n'), enumerable: options.debug }, query: { value: query.string, enumerable: options.debug }, diff --git a/cjs/src/index.js b/cjs/src/index.js index 40ac2c18..baf7e60a 100644 --- a/cjs/src/index.js +++ b/cjs/src/index.js @@ -204,9 +204,10 @@ function Postgres(a, b) { const queue = Queue() const c = open.length ? open.shift() - : await new Promise(r => { - queries.push({ reserve: r }) - closed.length && connect(closed.shift()) + : await new Promise((resolve, reject) => { + const query = { reserve: resolve, reject } + queries.push(query) + closed.length && connect(closed.shift(), query) }) move(c, reserved) @@ -480,7 +481,7 @@ function parseOptions(a, b) { {} ), connection : { - application_name: 'postgres.js', + application_name: env.PGAPPNAME || 'postgres.js', ...o.connection, ...Object.entries(query).reduce((acc, [k, v]) => (k in defaults || (acc[k] = v), acc), {}) }, diff --git a/cjs/tests/index.js b/cjs/tests/index.js index 7d84ac67..901edb54 100644 --- a/cjs/tests/index.js +++ b/cjs/tests/index.js @@ -429,6 +429,30 @@ t('Reconnect using SSL', { timeout: 2 }, async() => { return [1, (await sql`select 1 as x`)[0].x] }) +t('Proper handling of non object Errors', async() => { + const sql = postgres({ socket: () => { throw 'wat' } }) // eslint-disable-line + + return [ + 'wat', await sql`select 1 as x`.catch(e => e.message) + ] +}) + +t('Proper handling of null Errors', async() => { + const sql = postgres({ socket: () => { throw null } }) // eslint-disable-line + + return [ + 'null', await sql`select 1 as x`.catch(e => e.message) + ] +}) + +t('Ensure reserve throws proper error', async() => { + const sql = postgres({ socket: () => { throw 'wat' }, idle_timeout }) // eslint-disable-line + + return [ + 'wat', await sql.reserve().catch(e => e) + ] +}) + t('Login without password', async() => { return [true, (await postgres({ ...options, ...login })`select true as x`)[0].x] }) diff --git a/deno/README.md b/deno/README.md index 6f8085cf..b6ec85b7 100644 --- a/deno/README.md +++ b/deno/README.md @@ -338,6 +338,27 @@ select * from users select * from users where user_id = $1 ``` +### Dynamic ordering + +```js +const id = 1 +const order = { + username: 'asc' + created_at: 'desc' +} +await sql` + select + * + from ticket + where account = ${ id } + order by ${ + Object.entries(order).flatMap(([column, order], i) => + [i ? sql`,` : sql``, sql`${ sql(column) } ${ order === 'desc' ? sql`desc` : sql`asc` }`] + ) + } +` +``` + ### SQL functions Using keywords or calling functions dynamically is also possible by using ``` sql`` ``` fragments. ```js @@ -564,6 +585,8 @@ If you know what you're doing, you can use `unsafe` to pass any string you'd lik sql.unsafe('select ' + danger + ' from users where id = ' + dragons) ``` +By default, `sql.unsafe` assumes the `query` string is sufficiently dynamic that prepared statements do not make sense, and so defaults them to off. If you'd like to re-enable prepared statements, you can pass `{ prepare: true }`. + You can also nest `sql.unsafe` within a safe `sql` expression. This is useful if only part of your fraction has unsafe elements. ```js @@ -1121,20 +1144,25 @@ It is also possible to connect to the database without a connection string or an const sql = postgres() ``` -| Option | Environment Variables | -| ----------------- | ------------------------ | -| `host` | `PGHOST` | -| `port` | `PGPORT` | -| `database` | `PGDATABASE` | -| `username` | `PGUSERNAME` or `PGUSER` | -| `password` | `PGPASSWORD` | -| `idle_timeout` | `PGIDLE_TIMEOUT` | -| `connect_timeout` | `PGCONNECT_TIMEOUT` | +| Option | Environment Variables | +| ------------------ | ------------------------ | +| `host` | `PGHOST` | +| `port` | `PGPORT` | +| `database` | `PGDATABASE` | +| `username` | `PGUSERNAME` or `PGUSER` | +| `password` | `PGPASSWORD` | +| `application_name` | `PGAPPNAME` | +| `idle_timeout` | `PGIDLE_TIMEOUT` | +| `connect_timeout` | `PGCONNECT_TIMEOUT` | ### Prepared statements Prepared statements will automatically be created for any queries where it can be inferred that the query is static. This can be disabled by using the `prepare: false` option. For instance — this is useful when [using PGBouncer in `transaction mode`](https://github.com/porsager/postgres/issues/93#issuecomment-656290493). +**update**: [since 1.21.0](https://www.pgbouncer.org/2023/10/pgbouncer-1-21-0) +PGBouncer supports protocol-level named prepared statements when [configured +properly](https://www.pgbouncer.org/config.html#max_prepared_statements) + ## Custom Types You can add ergonomic support for custom types, or simply use `sql.typed(value, type)` inline, where type is the PostgreSQL `oid` for the type and the correctly serialized string. _(`oid` values for types can be found in the `pg_catalog.pg_type` table.)_ @@ -1294,8 +1322,8 @@ This error is thrown if the user has called [`sql.end()`](#teardown--cleanup) an This error is thrown for any queries that were pending when the timeout to [`sql.end({ timeout: X })`](#teardown--cleanup) was reached. -##### CONNECTION_CONNECT_TIMEOUT -> write CONNECTION_CONNECT_TIMEOUT host:port +##### CONNECT_TIMEOUT +> write CONNECT_TIMEOUT host:port This error is thrown if the startup phase of the connection (tcp, protocol negotiation, and auth) took more than the default 30 seconds or what was specified using `connect_timeout` or `PGCONNECT_TIMEOUT`. diff --git a/deno/src/connection.js b/deno/src/connection.js index 1726a9aa..79c64be5 100644 --- a/deno/src/connection.js +++ b/deno/src/connection.js @@ -296,7 +296,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose if (incomings) { incomings.push(x) remaining -= x.length - if (remaining >= 0) + if (remaining > 0) return } @@ -384,10 +384,13 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose function errored(err) { stream && (stream.destroy(err), stream = null) query && queryError(query, err) - initial && (queryError(initial, err), initial = null) + initial && (initial.reserve ? initial.reject(err) : queryError(initial, err), initial = null) } function queryError(query, err) { + if (!err || typeof err !== 'object') + err = new Error(err) + 'query' in err || 'parameters' in err || Object.defineProperties(err, { stack: { value: err.stack + query.origin.replace(/.*\n/, '\n'), enumerable: options.debug }, query: { value: query.string, enumerable: options.debug }, diff --git a/deno/src/index.js b/deno/src/index.js index 3bbdf2ba..98a82345 100644 --- a/deno/src/index.js +++ b/deno/src/index.js @@ -205,9 +205,10 @@ function Postgres(a, b) { const queue = Queue() const c = open.length ? open.shift() - : await new Promise(r => { - queries.push({ reserve: r }) - closed.length && connect(closed.shift()) + : await new Promise((resolve, reject) => { + const query = { reserve: resolve, reject } + queries.push(query) + closed.length && connect(closed.shift(), query) }) move(c, reserved) @@ -481,7 +482,7 @@ function parseOptions(a, b) { {} ), connection : { - application_name: 'postgres.js', + application_name: env.PGAPPNAME || 'postgres.js', ...o.connection, ...Object.entries(query).reduce((acc, [k, v]) => (k in defaults || (acc[k] = v), acc), {}) }, diff --git a/deno/tests/index.js b/deno/tests/index.js index 5b5d6e57..45c3421b 100644 --- a/deno/tests/index.js +++ b/deno/tests/index.js @@ -431,6 +431,30 @@ t('Reconnect using SSL', { timeout: 2 }, async() => { return [1, (await sql`select 1 as x`)[0].x] }) +t('Proper handling of non object Errors', async() => { + const sql = postgres({ socket: () => { throw 'wat' } }) // eslint-disable-line + + return [ + 'wat', await sql`select 1 as x`.catch(e => e.message) + ] +}) + +t('Proper handling of null Errors', async() => { + const sql = postgres({ socket: () => { throw null } }) // eslint-disable-line + + return [ + 'null', await sql`select 1 as x`.catch(e => e.message) + ] +}) + +t('Ensure reserve throws proper error', async() => { + const sql = postgres({ socket: () => { throw 'wat' }, idle_timeout }) // eslint-disable-line + + return [ + 'wat', await sql.reserve().catch(e => e) + ] +}) + t('Login without password', async() => { return [true, (await postgres({ ...options, ...login })`select true as x`)[0].x] }) diff --git a/deno/types/index.d.ts b/deno/types/index.d.ts index 2088662d..44a07af0 100644 --- a/deno/types/index.d.ts +++ b/deno/types/index.d.ts @@ -458,7 +458,8 @@ declare namespace postgres { | 'NOT_TAGGED_CALL' | 'UNDEFINED_VALUE' | 'MAX_PARAMETERS_EXCEEDED' - | 'SASL_SIGNATURE_MISMATCH'; + | 'SASL_SIGNATURE_MISMATCH' + | 'UNSAFE_TRANSACTION'; message: string; } diff --git a/tests/index.js b/tests/index.js index 5b5e6f87..7a8afbf4 100644 --- a/tests/index.js +++ b/tests/index.js @@ -430,7 +430,7 @@ t('Reconnect using SSL', { timeout: 2 }, async() => { }) t('Proper handling of non object Errors', async() => { - const sql = postgres({ socket: () => { throw 'wat' } }) + const sql = postgres({ socket: () => { throw 'wat' } }) // eslint-disable-line return [ 'wat', await sql`select 1 as x`.catch(e => e.message) @@ -438,7 +438,7 @@ t('Proper handling of non object Errors', async() => { }) t('Proper handling of null Errors', async() => { - const sql = postgres({ socket: () => { throw null } }) + const sql = postgres({ socket: () => { throw null } }) // eslint-disable-line return [ 'null', await sql`select 1 as x`.catch(e => e.message) @@ -446,7 +446,7 @@ t('Proper handling of null Errors', async() => { }) t('Ensure reserve throws proper error', async() => { - const sql = postgres({ socket: () => { throw 'wat' }, idle_timeout }) + const sql = postgres({ socket: () => { throw 'wat' }, idle_timeout }) // eslint-disable-line return [ 'wat', await sql.reserve().catch(e => e) From ca4da7ca56a5fad7a4fd495c9e640c66f325bc45 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Tue, 20 May 2025 17:40:43 -0600 Subject: [PATCH 83/88] 3.4.6 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f556cd28..0748f3b8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "postgres", - "version": "3.4.5", + "version": "3.4.6", "description": "Fastest full featured PostgreSQL client for Node.js", "type": "module", "module": "src/index.js", From e34826d43036bda349a18d0354389ec6d737aec4 Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Wed, 21 May 2025 09:59:53 -0600 Subject: [PATCH 84/88] Fix reserved queries failing on connect --- src/connection.js | 11 +++++++---- tests/index.js | 12 +++++++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/connection.js b/src/connection.js index b1ff1244..c3f554aa 100644 --- a/src/connection.js +++ b/src/connection.js @@ -109,7 +109,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose queue: queues.closed, idleTimer, connect(query) { - initial = query || true + initial = query reconnect() }, terminate, @@ -381,10 +381,13 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose function errored(err) { stream && (stream.destroy(err), stream = null) query && queryError(query, err) - initial && (initial.reserve ? initial.reject(err) : queryError(initial, err), initial = null) + initial && (queryError(initial, err), initial = null) } function queryError(query, err) { + if (query.reserve) + return query.reject(err) + if (!err || typeof err !== 'object') err = new Error(err) @@ -535,11 +538,11 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } if (needsTypes) { - initial === true && (initial = null) + initial.reserve && (initial = null) return fetchArrayTypes() } - initial !== true && execute(initial) + initial && !initial.reserve && execute(initial) options.shared.retries = retries = 0 initial = null return diff --git a/tests/index.js b/tests/index.js index 7a8afbf4..07ff98ed 100644 --- a/tests/index.js +++ b/tests/index.js @@ -445,7 +445,7 @@ t('Proper handling of null Errors', async() => { ] }) -t('Ensure reserve throws proper error', async() => { +t('Ensure reserve on connection throws proper error', async() => { const sql = postgres({ socket: () => { throw 'wat' }, idle_timeout }) // eslint-disable-line return [ @@ -2604,3 +2604,13 @@ t('arrays in reserved connection', async() => { x.join('') ] }) + +t('Ensure reserve on query throws proper error', async() => { + const sql = postgres({ idle_timeout }) // eslint-disable-line + const reserved = await sql.reserve() + const [{ x }] = await reserved`select 'wat' as x` + + return [ + 'wat', x, reserved.release() + ] +}) From 657fea0959ed0ffc228f851bbff13b13e6179eef Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Wed, 21 May 2025 10:00:41 -0600 Subject: [PATCH 85/88] build --- cf/src/connection.js | 11 +++++++---- cjs/src/connection.js | 11 +++++++---- cjs/tests/index.js | 12 +++++++++++- deno/src/connection.js | 11 +++++++---- deno/tests/index.js | 12 +++++++++++- 5 files changed, 43 insertions(+), 14 deletions(-) diff --git a/cf/src/connection.js b/cf/src/connection.js index d918b136..203af80d 100644 --- a/cf/src/connection.js +++ b/cf/src/connection.js @@ -111,7 +111,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose queue: queues.closed, idleTimer, connect(query) { - initial = query || true + initial = query reconnect() }, terminate, @@ -383,10 +383,13 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose function errored(err) { stream && (stream.destroy(err), stream = null) query && queryError(query, err) - initial && (initial.reserve ? initial.reject(err) : queryError(initial, err), initial = null) + initial && (queryError(initial, err), initial = null) } function queryError(query, err) { + if (query.reserve) + return query.reject(err) + if (!err || typeof err !== 'object') err = new Error(err) @@ -537,11 +540,11 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } if (needsTypes) { - initial === true && (initial = null) + initial.reserve && (initial = null) return fetchArrayTypes() } - initial !== true && execute(initial) + initial && !initial.reserve && execute(initial) options.shared.retries = retries = 0 initial = null return diff --git a/cjs/src/connection.js b/cjs/src/connection.js index db427247..589d3638 100644 --- a/cjs/src/connection.js +++ b/cjs/src/connection.js @@ -109,7 +109,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose queue: queues.closed, idleTimer, connect(query) { - initial = query || true + initial = query reconnect() }, terminate, @@ -381,10 +381,13 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose function errored(err) { stream && (stream.destroy(err), stream = null) query && queryError(query, err) - initial && (initial.reserve ? initial.reject(err) : queryError(initial, err), initial = null) + initial && (queryError(initial, err), initial = null) } function queryError(query, err) { + if (query.reserve) + return query.reject(err) + if (!err || typeof err !== 'object') err = new Error(err) @@ -535,11 +538,11 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } if (needsTypes) { - initial === true && (initial = null) + initial.reserve && (initial = null) return fetchArrayTypes() } - initial !== true && execute(initial) + initial && !initial.reserve && execute(initial) options.shared.retries = retries = 0 initial = null return diff --git a/cjs/tests/index.js b/cjs/tests/index.js index 901edb54..ec5222f7 100644 --- a/cjs/tests/index.js +++ b/cjs/tests/index.js @@ -445,7 +445,7 @@ t('Proper handling of null Errors', async() => { ] }) -t('Ensure reserve throws proper error', async() => { +t('Ensure reserve on connection throws proper error', async() => { const sql = postgres({ socket: () => { throw 'wat' }, idle_timeout }) // eslint-disable-line return [ @@ -2604,3 +2604,13 @@ t('arrays in reserved connection', async() => { x.join('') ] }) + +t('Ensure reserve on query throws proper error', async() => { + const sql = postgres({ idle_timeout }) // eslint-disable-line + const reserved = await sql.reserve() + const [{ x }] = await reserved`select 'wat' as x` + + return [ + 'wat', x, reserved.release() + ] +}) diff --git a/deno/src/connection.js b/deno/src/connection.js index 79c64be5..a3f43c48 100644 --- a/deno/src/connection.js +++ b/deno/src/connection.js @@ -112,7 +112,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose queue: queues.closed, idleTimer, connect(query) { - initial = query || true + initial = query reconnect() }, terminate, @@ -384,10 +384,13 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose function errored(err) { stream && (stream.destroy(err), stream = null) query && queryError(query, err) - initial && (initial.reserve ? initial.reject(err) : queryError(initial, err), initial = null) + initial && (queryError(initial, err), initial = null) } function queryError(query, err) { + if (query.reserve) + return query.reject(err) + if (!err || typeof err !== 'object') err = new Error(err) @@ -538,11 +541,11 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose } if (needsTypes) { - initial === true && (initial = null) + initial.reserve && (initial = null) return fetchArrayTypes() } - initial !== true && execute(initial) + initial && !initial.reserve && execute(initial) options.shared.retries = retries = 0 initial = null return diff --git a/deno/tests/index.js b/deno/tests/index.js index 45c3421b..adedf1e0 100644 --- a/deno/tests/index.js +++ b/deno/tests/index.js @@ -447,7 +447,7 @@ t('Proper handling of null Errors', async() => { ] }) -t('Ensure reserve throws proper error', async() => { +t('Ensure reserve on connection throws proper error', async() => { const sql = postgres({ socket: () => { throw 'wat' }, idle_timeout }) // eslint-disable-line return [ @@ -2607,4 +2607,14 @@ t('arrays in reserved connection', async() => { ] }) +t('Ensure reserve on query throws proper error', async() => { + const sql = postgres({ idle_timeout }) // eslint-disable-line + const reserved = await sql.reserve() + const [{ x }] = await reserved`select 'wat' as x` + + return [ + 'wat', x, reserved.release() + ] +}) + ;globalThis.addEventListener("unload", () => Deno.exit(process.exitCode)) \ No newline at end of file From 0068aa4825e1087cdad16f37142b447b3f0eab0c Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Wed, 21 May 2025 10:03:50 -0600 Subject: [PATCH 86/88] Test node 23 and 24 too --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index af00f7e0..970d2771 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - node: ['12', '14', '16', '18', '20', '21', '22'] + node: ['12', '14', '16', '18', '20', '21', '22', '23', '24'] postgres: ['12', '13', '14', '15', '16', '17'] runs-on: ubuntu-latest services: From 9b92b65da6a5121545581a6dd5de859c2a70177f Mon Sep 17 00:00:00 2001 From: Rasmus Porsager Date: Wed, 21 May 2025 10:13:48 -0600 Subject: [PATCH 87/88] 3.4.7 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0748f3b8..65157609 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "postgres", - "version": "3.4.6", + "version": "3.4.7", "description": "Fastest full featured PostgreSQL client for Node.js", "type": "module", "module": "src/index.js", From 32feb259a3c9abffab761bd1758b3168d9e0cebc Mon Sep 17 00:00:00 2001 From: Jobians Techie <88005779+Jobians@users.noreply.github.com> Date: Thu, 12 Jun 2025 23:18:42 +0100 Subject: [PATCH 88/88] Fix PGAPPNAME env access by prioritizing connection.application_name Use connection.application_name if provided before falling back to PGAPPNAME env var to avoid unnecessary env access errors. --- deno/src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deno/src/index.js b/deno/src/index.js index 98a82345..aa7a920f 100644 --- a/deno/src/index.js +++ b/deno/src/index.js @@ -482,8 +482,8 @@ function parseOptions(a, b) { {} ), connection : { - application_name: env.PGAPPNAME || 'postgres.js', ...o.connection, + application_name: o.connection?.application_name ?? env.PGAPPNAME ?? 'postgres.js', ...Object.entries(query).reduce((acc, [k, v]) => (k in defaults || (acc[k] = v), acc), {}) }, types : o.types || {},