From 5bc209b6e06422fd6f5187c7b8bfb2e61de63298 Mon Sep 17 00:00:00 2001 From: bombillazo Date: Sun, 18 Feb 2024 02:26:24 -0400 Subject: [PATCH 1/6] feat: add logic to handle array types for custom decoders --- connection/connection_params.ts | 12 +++++- query/array_parser.ts | 10 +++++ query/decode.ts | 24 +++++++++-- tests/query_client_test.ts | 73 +++++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 5 deletions(-) diff --git a/connection/connection_params.ts b/connection/connection_params.ts index 7b68ea9c..cc3971f7 100644 --- a/connection/connection_params.ts +++ b/connection/connection_params.ts @@ -3,6 +3,7 @@ import { ConnectionParamsError } from "../client/error.ts"; import { fromFileUrl, isAbsolute } from "../deps.ts"; import { OidType } from "../query/oid.ts"; import { DebugControls } from "../debug.ts"; +import { ParseArrayFunc } from "../query/array_parser.ts"; /** * The connection string must match the following URI structure. All parameters but database and user are optional @@ -108,9 +109,16 @@ export type Decoders = { /** * A decoder function that takes a string value and returns a parsed value of some type. - * the Oid is also passed to the function for reference + * + * @param value The string value to parse + * @param oid The OID of the column type the value is from + * @param parseArray A helper function that parses SQL array-formatted strings and parses each array value using a transform function. */ -export type DecoderFunction = (value: string, oid: number) => unknown; +export type DecoderFunction = ( + value: string, + oid: number, + parseArray: ParseArrayFunc, +) => unknown; /** * Control the behavior for the client instance diff --git a/query/array_parser.ts b/query/array_parser.ts index 9fd043bd..312d0406 100644 --- a/query/array_parser.ts +++ b/query/array_parser.ts @@ -6,6 +6,16 @@ type AllowedSeparators = "," | ";"; type ArrayResult = Array>; type Transformer = (value: string) => T; +export type ParseArrayFunc = typeof parseArray; + +/** + * Parse a string into an array of values using the provided transform function. + * + * @param source The string to parse + * @param transform A function to transform each value in the array + * @param separator The separator used to split the string into values + * @returns + */ export function parseArray( source: string, transform: Transformer, diff --git a/query/decode.ts b/query/decode.ts index c2b5ec42..fb13afa3 100644 --- a/query/decode.ts +++ b/query/decode.ts @@ -1,4 +1,4 @@ -import { Oid, OidTypes, OidValue } from "./oid.ts"; +import { Oid, OidType, OidTypes, OidValue } from "./oid.ts"; import { bold, yellow } from "../deps.ts"; import { decodeBigint, @@ -36,6 +36,7 @@ import { decodeTidArray, } from "./decoders.ts"; import { ClientControls } from "../connection/connection_params.ts"; +import { parseArray } from "./array_parser.ts"; export class Column { constructor( @@ -216,12 +217,29 @@ export function decode( // check if there is a custom decoder if (controls?.decoders) { + const oidType = OidTypes[column.typeOid as OidValue]; // check if there is a custom decoder by oid (number) or by type name (string) const decoderFunc = controls.decoders?.[column.typeOid] || - controls.decoders?.[OidTypes[column.typeOid as OidValue]]; + controls.decoders?.[oidType]; if (decoderFunc) { - return decoderFunc(strValue, column.typeOid); + return decoderFunc(strValue, column.typeOid, parseArray); + } // if no custom decoder is found and the oid is for an array type, check if there is + // a decoder for the base type and use that with the array parser + else if (oidType.includes("_array")) { + const baseOidType = oidType.replace("_array", "") as OidType; + // check if the base type is in the Oid object + if (baseOidType in Oid) { + // check if there is a custom decoder for the base type by oid (number) or by type name (string) + const decoderFunc = controls.decoders?.[Oid[baseOidType]] || + controls.decoders?.[baseOidType]; + if (decoderFunc) { + return parseArray( + strValue, + (value: string) => decoderFunc(value, column.typeOid, parseArray), + ); + } + } } } diff --git a/tests/query_client_test.ts b/tests/query_client_test.ts index 0e71da69..c096049a 100644 --- a/tests/query_client_test.ts +++ b/tests/query_client_test.ts @@ -241,6 +241,79 @@ Deno.test( ), ); +Deno.test( + "Custom decoders with arrays", + withClient( + async (client) => { + const result = await client.queryObject( + `SELECT + ARRAY[true, false, true] AS _bool_array, + ARRAY['2024-01-01'::date, '2024-01-02'::date, '2024-01-03'::date] AS _date_array, + ARRAY[1.5:: REAL, 2.5::REAL, 3.5::REAL] AS _float_array, + ARRAY[10, 20, 30] AS _int_array, + ARRAY[ + '{"key1": "value1", "key2": "value2"}'::jsonb, + '{"key3": "value3", "key4": "value4"}'::jsonb, + '{"key5": "value5", "key6": "value6"}'::jsonb + ] AS _jsonb_array, + ARRAY['string1', 'string2', 'string3'] AS _text_array + ;`, + ); + + assertEquals(result.rows, [ + { + _bool_array: [ + { boolean: true }, + { boolean: false }, + { boolean: true }, + ], + _date_array: [ + new Date("2024-01-11T00:00:00.000Z"), + new Date("2024-01-12T00:00:00.000Z"), + new Date("2024-01-13T00:00:00.000Z"), + ], + _float_array: [15, 25, 35], + _int_array: [110, 120, 130], + _jsonb_array: [ + { key1: "value1", key2: "value2" }, + { key3: "value3", key4: "value4" }, + { key5: "value5", key6: "value6" }, + ], + _text_array: ["string1_!", "string2_!", "string3_!"], + }, + ]); + }, + { + controls: { + decoders: { + // convert to object + [Oid.bool]: (value: string) => ({ boolean: value === "t" }), + // 1082 = date : convert to date and add 10 days + "1082": (value: string) => { + const d = new Date(value); + return new Date(d.setDate(d.getDate() + 10)); + }, + // multiply by 20, should not be used! + float4: (value: string) => parseFloat(value) * 20, + // multiply by 10 + float4_array: (value: string, _, parseArray) => + parseArray(value, (v) => parseFloat(v) * 10), + // return 0, should not be used! + [Oid.int4]: () => 0, + // add 100 + [Oid.int4_array]: (value: string, _, parseArray) => + parseArray(value, (v) => parseInt(v, 10) + 100), + // split string and reverse, should not be used! + [Oid.text]: (value: string) => value.split("").reverse(), + // 1009 = text_array : append "_!" to each string + 1009: (value: string, _, parseArray) => + parseArray(value, (v) => `${v}_!`), + }, + }, + }, + ), +); + Deno.test( "Custom decoder precedence", withClient( From ad3e8f9d5f91f4dfbce999c522125ba3ef3c5c00 Mon Sep 17 00:00:00 2001 From: bombillazo Date: Sun, 18 Feb 2024 02:40:53 -0400 Subject: [PATCH 2/6] chore: update docs --- docs/README.md | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/README.md b/docs/README.md index 477b86f4..fc802f37 100644 --- a/docs/README.md +++ b/docs/README.md @@ -758,10 +758,10 @@ available: You can also provide custom decoders to the client that will be used to decode the result data. This can be done by setting the `decoders` controls option in the client configuration. This option is a map object where the keys are the -type names or Oid numbers and the values are the custom decoder functions. +type names or OID numbers and the values are the custom decoder functions. You can use it with the decode strategy. Custom decoders take precedence over -the strategy and internal parsers. +the strategy and internal decoders. ```ts { @@ -785,7 +785,36 @@ the strategy and internal parsers. const result = await client.queryObject( "SELECT ID, NAME, IS_ACTIVE FROM PEOPLE", ); - console.log(result.rows[0]); // {id: '1', name: 'Javier', is_active: { value: false, type: "boolean"}} + console.log(result.rows[0]); + // {id: '1', name: 'Javier', is_active: { value: false, type: "boolean"}} +} +``` + +The driver takes care of parsing the related `array` OID types automatically. +For example, if a custom decoder is defined for the `int4` type, it will be applied +when parsing `int4[]` arrays. If needed, you can have separate custom decoders for the +array and non-array types by defining another custom decoders for the array type itself. + +```ts +{ + const client = new Client({ + database: "some_db", + user: "some_user", + controls: { + decodeStrategy: "string", + decoders: { + // Custom decoder for int4 (OID 23 = int4) + // convert to int and multiply by 100 + 23: (value: string) => parseInt(value, 10) * 100, + }, + }, + }); + + const result = await client.queryObject( + "SELECT ARRAY[ 2, 2, 3, 1 ] AS scores, 8 final_score;", + ); + console.log(result.rows[0]); + // { scores: [ 200, 200, 300, 100 ], final_score: 800 } } ``` From cc87d8c31d4d7e126a79c2b724f31f756825948c Mon Sep 17 00:00:00 2001 From: bombillazo Date: Sun, 18 Feb 2024 02:43:01 -0400 Subject: [PATCH 3/6] chore: fix format --- docs/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/README.md b/docs/README.md index fc802f37..c4763079 100644 --- a/docs/README.md +++ b/docs/README.md @@ -791,9 +791,10 @@ the strategy and internal decoders. ``` The driver takes care of parsing the related `array` OID types automatically. -For example, if a custom decoder is defined for the `int4` type, it will be applied -when parsing `int4[]` arrays. If needed, you can have separate custom decoders for the -array and non-array types by defining another custom decoders for the array type itself. +For example, if a custom decoder is defined for the `int4` type, it will be +applied when parsing `int4[]` arrays. If needed, you can have separate custom +decoders for the array and non-array types by defining another custom decoders +for the array type itself. ```ts { From 9955bf74dbaa3e9418f91eeb97616ebbcb2b6481 Mon Sep 17 00:00:00 2001 From: bombillazo Date: Sun, 18 Feb 2024 02:45:55 -0400 Subject: [PATCH 4/6] chore: bump version, fix type name --- connection/connection_params.ts | 4 ++-- deno.json | 2 +- query/array_parser.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/connection/connection_params.ts b/connection/connection_params.ts index cc3971f7..ac4f650e 100644 --- a/connection/connection_params.ts +++ b/connection/connection_params.ts @@ -3,7 +3,7 @@ import { ConnectionParamsError } from "../client/error.ts"; import { fromFileUrl, isAbsolute } from "../deps.ts"; import { OidType } from "../query/oid.ts"; import { DebugControls } from "../debug.ts"; -import { ParseArrayFunc } from "../query/array_parser.ts"; +import { ParseArrayFunction } from "../query/array_parser.ts"; /** * The connection string must match the following URI structure. All parameters but database and user are optional @@ -117,7 +117,7 @@ export type Decoders = { export type DecoderFunction = ( value: string, oid: number, - parseArray: ParseArrayFunc, + parseArray: ParseArrayFunction, ) => unknown; /** diff --git a/deno.json b/deno.json index 10162a4f..a95580a3 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "lock": false, "name": "@bartlomieju/postgres", - "version": "0.18.1", + "version": "0.19.1", "exports": "./mod.ts" } diff --git a/query/array_parser.ts b/query/array_parser.ts index 312d0406..b7983b41 100644 --- a/query/array_parser.ts +++ b/query/array_parser.ts @@ -6,7 +6,7 @@ type AllowedSeparators = "," | ";"; type ArrayResult = Array>; type Transformer = (value: string) => T; -export type ParseArrayFunc = typeof parseArray; +export type ParseArrayFunction = typeof parseArray; /** * Parse a string into an array of values using the provided transform function. From 7b87470001c7cf27bdec6ac5bba771b8aba6a311 Mon Sep 17 00:00:00 2001 From: bombillazo Date: Sun, 18 Feb 2024 02:51:27 -0400 Subject: [PATCH 5/6] chore: update test readme --- tests/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/README.md b/tests/README.md index 10a1a496..e97e1563 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,9 +1,11 @@ # Testing -To run tests, first prepare your configuration file by copying +To run tests, we recommend using Docker. With Docker, there is no +need to modify any configuration, just run the build and test commands. + +If running tests on your host, prepare your configuration file by copying `config.example.json` into `config.json` and updating it appropriately based on -your environment. If you use the Docker based configuration below there's no -need to modify the configuration. +your environment. ## Running the Tests From 1e4827dc4225c4c5d3c2516cfc42e69b3396dfc3 Mon Sep 17 00:00:00 2001 From: bombillazo Date: Sun, 18 Feb 2024 02:54:34 -0400 Subject: [PATCH 6/6] chore: format readme --- tests/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/README.md b/tests/README.md index e97e1563..c8c3e4e9 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,7 +1,7 @@ # Testing -To run tests, we recommend using Docker. With Docker, there is no -need to modify any configuration, just run the build and test commands. +To run tests, we recommend using Docker. With Docker, there is no need to modify +any configuration, just run the build and test commands. If running tests on your host, prepare your configuration file by copying `config.example.json` into `config.json` and updating it appropriately based on