From 4cc974065e6c843457581cfe4d2443bccba28765 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 07:40:17 +0100 Subject: [PATCH 1/8] Bump ktor-network from 2.1.2 to 2.1.3 (#98) --- postgres-native-sqldelight-driver/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-native-sqldelight-driver/build.gradle.kts b/postgres-native-sqldelight-driver/build.gradle.kts index 2cd012d..807bbf3 100644 --- a/postgres-native-sqldelight-driver/build.gradle.kts +++ b/postgres-native-sqldelight-driver/build.gradle.kts @@ -33,7 +33,7 @@ kotlin { sourceSets { commonMain { dependencies { - api("io.ktor:ktor-network:2.1.2") + api("io.ktor:ktor-network:2.1.3") api("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4") api("app.cash.sqldelight:runtime:2.0.0-alpha04") api("org.jetbrains.kotlinx:kotlinx-datetime:0.4.0") From 80e8997c55662d806659083f12bea56db84b9bd9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Nov 2022 07:21:05 +0100 Subject: [PATCH 2/8] Bump multiplatform from 1.7.20 to 1.7.21 (#99) --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 22f1573..a58fe35 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -3,7 +3,7 @@ import org.jetbrains.kotlin.gradle.dsl.* import java.util.* plugins { - kotlin("multiplatform") version "1.7.20" apply false + kotlin("multiplatform") version "1.7.21" apply false `maven-publish` signing id("io.github.gradle-nexus.publish-plugin") version "1.1.0" From 98ddde69c10e38305940fc61676c790fd03ca8b8 Mon Sep 17 00:00:00 2001 From: Philip Wedemann <22521688+hfhbd@users.noreply.github.com> Date: Fri, 18 Nov 2022 16:11:13 +0100 Subject: [PATCH 3/8] Add executeQueryAsFlow and publish Postgres types in nativeCursor method (#106) Co-authored-by: hfhbd --- README.md | 66 +++++++++- .../sqldelight/postgresdriver/NoCursor.kt | 3 +- .../postgresdriver/PostgresCursor.kt | 2 +- .../postgresdriver/PostgresNativeDriver.kt | 120 +++++++++++------- .../sqldelight/postgresdriver/RealCursor.kt | 3 +- .../PostgresNativeDriverTest.kt | 18 ++- 6 files changed, 159 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 13f9ad8..de0494c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # Module postgres-native-sqldelight -A native Postgres driver for SqlDelight. +A native Postgres driver using libpq. + +You can use the driver with [SQLDelight](https://github.com/cashapp/sqldelight), but this is not required. - [Source code](https://github.com/hfhbd/postgres-native-sqldelight) @@ -47,7 +49,67 @@ Local listeners only notify this client, ideally for testing or using the databa SQLDelight only. Remote listener support uses `NOTIFY` and `LISTEN`, so you can use this with multiple clients or with existing database triggers. -SQLDelight uses and expects the table name as payload, but you can provide a mapper function. +SQLDelight uses and expects the table name as payload, but you can provide a mapper function. + +### SQLDelight Support + +Just create the driver and use your database instances in the usual way. + +### Raw usage + +Beside SQLDelight you could also use this driver with raw queries. +The identifier is used to reuse prepared statements. + +```kotlin +driver.execute(identifier = null, sql = "INSERT INTO foo VALUES (42)", parameters = 0, binders = null) +``` + +It also supports a real lazy cursor or a flow: + +```kotlin +val names: List = driver.executeQueryWithNativeCursor( + identifier = null, + sql = "SELECT name from foo", + mapper = { cursor -> + buildList { + while (cursor.next()) { + add( + Simple( + index = cursor.getLong(0)!!.toInt(), + name = cursor.getString(1), + byteArray = cursor.getBytes(2) + ) + ) + } + } + }, + parameters = 0, + fetchSize = 100, + binders = null +) + +val namesFlow: Flow = driver.executeQueryAsFlow( + identifier = null, + sql = "SELECT name from foo", + mapper = { cursor -> + Simple( + index = cursor.getLong(0)!!.toInt(), + name = cursor.getString(1), + byteArray = cursor.getBytes(2) + ) + }, + parameters = 0, + fetchSize = 100, + binders = null +) +``` + +And for bulk imports, use the `copy` method: + +```kotlin +driver.execute(514394779, """COPY foo FROM STDIN (FORMAT CSV)""", 0) +val rows = driver.copy("1,2,3\n4,5,6\n") +``` ## License diff --git a/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/NoCursor.kt b/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/NoCursor.kt index bcb6e19..ca8e09f 100644 --- a/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/NoCursor.kt +++ b/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/NoCursor.kt @@ -1,11 +1,12 @@ package app.softwork.sqldelight.postgresdriver +import app.cash.sqldelight.db.* import kotlinx.cinterop.* import libpq.* internal class NoCursor( result: CPointer -) : PostgresCursor(result) { +) : PostgresCursor(result), Closeable { override fun close() { result.clear() } diff --git a/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/PostgresCursor.kt b/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/PostgresCursor.kt index 2055ee4..0defd6f 100644 --- a/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/PostgresCursor.kt +++ b/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/PostgresCursor.kt @@ -9,7 +9,7 @@ import kotlin.time.* public sealed class PostgresCursor( internal var result: CPointer -) : SqlCursor, Closeable { +) : SqlCursor { internal abstract val currentRowIndex: Int override fun getBoolean(index: Int): Boolean? = getString(index)?.toBoolean() diff --git a/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/PostgresNativeDriver.kt b/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/PostgresNativeDriver.kt index 4df4312..629e995 100644 --- a/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/PostgresNativeDriver.kt +++ b/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/PostgresNativeDriver.kt @@ -176,7 +176,7 @@ public class PostgresNativeDriver( private fun preparedStatement( parameters: Int, - binders: (SqlPreparedStatement.() -> Unit)? + binders: (PostgresPreparedStatement.() -> Unit)? ): PostgresPreparedStatement? = if (parameters != 0) { PostgresPreparedStatement(parameters).apply { if (binders != null) { @@ -185,52 +185,6 @@ public class PostgresNativeDriver( } } else null - public fun executeQueryWithNativeCursor( - identifier: Int?, - sql: String, - mapper: (SqlCursor) -> R, - parameters: Int, - fetchSize: Int = 1, - binders: (SqlPreparedStatement.() -> Unit)? - ): QueryResult.Value { - val cursorName = if (identifier == null) "myCursor" else "cursor${identifier.escapeNegative()}" - val cursor = "DECLARE $cursorName CURSOR FOR" - - val preparedStatement = preparedStatement(parameters, binders) - val result = if (identifier != null) { - checkPreparedStatement(identifier, "$cursor $sql", parameters, preparedStatement) - conn.exec("BEGIN") - memScoped { - PQexecPrepared( - conn, - stmtName = identifier.toString(), - nParams = parameters, - paramValues = preparedStatement?.values(this), - paramLengths = preparedStatement?.lengths?.refTo(0), - paramFormats = preparedStatement?.formats?.refTo(0), - resultFormat = TEXT_RESULT_FORMAT - ) - } - } else { - conn.exec("BEGIN") - memScoped { - PQexecParams( - conn, - command = "$cursor $sql", - nParams = parameters, - paramValues = preparedStatement?.values(this), - paramLengths = preparedStatement?.lengths?.refTo(0), - paramFormats = preparedStatement?.formats?.refTo(0), - paramTypes = preparedStatement?.types?.refTo(0), - resultFormat = TEXT_RESULT_FORMAT - ) - } - }.check(conn) - - val value = RealCursor(result, cursorName, conn, fetchSize).use(mapper) - return QueryResult.Value(value = value) - } - private fun checkPreparedStatement( identifier: Int, sql: String, @@ -321,6 +275,8 @@ public class PostgresNativeDriver( } } + // Custom functions + public fun copy(stdin: String): Long { val status = PQputCopyData(conn, stdin, stdin.encodeToByteArray().size) check(status == 1) { @@ -333,6 +289,76 @@ public class PostgresNativeDriver( val result = PQgetResult(conn).check(conn) return result.rows } + + public fun executeQueryAsFlow( + identifier: Int?, + sql: String, + mapper: suspend (PostgresCursor) -> R, + parameters: Int, + fetchSize: Int = 1, + binders: (PostgresPreparedStatement.() -> Unit)? + ): Flow = flow { + val (result, cursorName) = prepareQuery(identifier, sql, parameters, binders) + RealCursor(result, cursorName, conn, fetchSize).use { + while (it.next()) { + emit(mapper(it)) + } + } + } + + private fun prepareQuery( + identifier: Int?, + sql: String, + parameters: Int, + binders: (PostgresPreparedStatement.() -> Unit)? + ): Pair, String> { + val cursorName = if (identifier == null) "myCursor" else "cursor${identifier.escapeNegative()}" + val cursor = "DECLARE $cursorName CURSOR FOR" + + val preparedStatement = preparedStatement(parameters, binders) + return if (identifier != null) { + checkPreparedStatement(identifier, "$cursor $sql", parameters, preparedStatement) + conn.exec("BEGIN") + memScoped { + PQexecPrepared( + conn, + stmtName = identifier.toString(), + nParams = parameters, + paramValues = preparedStatement?.values(this), + paramLengths = preparedStatement?.lengths?.refTo(0), + paramFormats = preparedStatement?.formats?.refTo(0), + resultFormat = TEXT_RESULT_FORMAT + ) + } + } else { + conn.exec("BEGIN") + memScoped { + PQexecParams( + conn, + command = "$cursor $sql", + nParams = parameters, + paramValues = preparedStatement?.values(this), + paramLengths = preparedStatement?.lengths?.refTo(0), + paramFormats = preparedStatement?.formats?.refTo(0), + paramTypes = preparedStatement?.types?.refTo(0), + resultFormat = TEXT_RESULT_FORMAT + ) + } + }.check(conn) to cursorName + } + + public fun executeQueryWithNativeCursor( + identifier: Int?, + sql: String, + mapper: (PostgresCursor) -> R, + parameters: Int, + fetchSize: Int = 1, + binders: (PostgresPreparedStatement.() -> Unit)? + ): QueryResult.Value { + val (result, cursorName) = prepareQuery(identifier, sql, parameters, binders) + val value = RealCursor(result, cursorName, conn, fetchSize).use(mapper) + return QueryResult.Value(value = value) + } } private fun CPointer?.error(): String { diff --git a/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/RealCursor.kt b/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/RealCursor.kt index caf4070..ea2b52d 100644 --- a/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/RealCursor.kt +++ b/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/RealCursor.kt @@ -1,5 +1,6 @@ package app.softwork.sqldelight.postgresdriver +import app.cash.sqldelight.db.* import kotlinx.cinterop.* import libpq.* @@ -11,7 +12,7 @@ internal class RealCursor( private val name: String, private val conn: CPointer, private val fetchSize: Int -) : PostgresCursor(result) { +) : PostgresCursor(result), Closeable { override fun close() { result.clear() conn.exec("CLOSE $name") diff --git a/postgres-native-sqldelight-driver/src/commonTest/kotlin/app/softwork/sqldelight/postgresdriver/PostgresNativeDriverTest.kt b/postgres-native-sqldelight-driver/src/commonTest/kotlin/app/softwork/sqldelight/postgresdriver/PostgresNativeDriverTest.kt index 53f65f4..f4097ec 100644 --- a/postgres-native-sqldelight-driver/src/commonTest/kotlin/app/softwork/sqldelight/postgresdriver/PostgresNativeDriverTest.kt +++ b/postgres-native-sqldelight-driver/src/commonTest/kotlin/app/softwork/sqldelight/postgresdriver/PostgresNativeDriverTest.kt @@ -10,7 +10,7 @@ import kotlin.time.Duration.Companion.seconds @ExperimentalCoroutinesApi class PostgresNativeDriverTest { @Test - fun simpleTest() { + fun simpleTest() = runTest { val driver = PostgresNativeDriver( host = "localhost", port = 5432, @@ -123,6 +123,22 @@ class PostgresNativeDriverTest { cursorList.size } + val cursorFlow = driver.executeQueryAsFlow( + -42, + "SELECT * FROM baz", + fetchSize = 1, + parameters = 0, + binders = null, + mapper = { + Simple( + index = it.getLong(0)!!.toInt(), + name = it.getString(1), + byteArray = it.getBytes(2) + ) + }) + assertEquals(7, cursorFlow.count()) + assertEquals(4, cursorFlow.take(4).count()) + expect(0) { val cursorList = driver.executeQueryWithNativeCursor( -100, From db8baddc641544dbd0247b34411cea4f5afcaeb5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Nov 2022 15:26:49 +0000 Subject: [PATCH 4/8] Bump idea from 222.4345.24 to 222.4459.20 (#104) * Bump idea from 222.4345.24 to 222.4459.20 Bumps `idea` from 222.4345.24 to 222.4459.20. Updates `core-impl` from 222.4345.24 to 222.4459.20 Updates `util-ui` from 222.4345.24 to 222.4459.20 Updates `project-model-impl` from 222.4345.24 to 222.4459.20 Updates `analysis-impl` from 222.4345.24 to 222.4459.20 --- updated-dependencies: - dependency-name: com.jetbrains.intellij.platform:core-impl dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: com.jetbrains.intellij.platform:util-ui dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: com.jetbrains.intellij.platform:project-model-impl dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: com.jetbrains.intellij.platform:analysis-impl dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Fix dependencies Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: hfhbd Co-authored-by: Philip Wedemann <22521688+hfhbd@users.noreply.github.com> --- postgres-native-sqldelight-dialect/build.gradle.kts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/postgres-native-sqldelight-dialect/build.gradle.kts b/postgres-native-sqldelight-dialect/build.gradle.kts index 49cce95..2fdf5a2 100644 --- a/postgres-native-sqldelight-dialect/build.gradle.kts +++ b/postgres-native-sqldelight-dialect/build.gradle.kts @@ -21,7 +21,7 @@ repositories { maven(url = "https://maven.pkg.jetbrains.space/public/p/ktor/eap") } -val idea = "222.4345.24" +val idea = "222.4459.20" grammarKit { intellijRelease.set(idea) @@ -61,6 +61,8 @@ configurations.all { exclude(group = "com.jetbrains.rd") exclude(group = "com.github.jetbrains", module = "jetCheck") exclude(group = "org.roaringbitmap") + exclude(group = "com.jetbrains.intellij.remoteDev") + exclude(group = "com.jetbrains.intellij.spellchecker") } tasks { From 174a4db2418d68551deb675c095a3e88007533dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Nov 2022 02:25:53 +0000 Subject: [PATCH 5/8] Bump idea from 222.4459.20 to 222.4459.23 (#107) --- postgres-native-sqldelight-dialect/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-native-sqldelight-dialect/build.gradle.kts b/postgres-native-sqldelight-dialect/build.gradle.kts index 2fdf5a2..b24a64e 100644 --- a/postgres-native-sqldelight-dialect/build.gradle.kts +++ b/postgres-native-sqldelight-dialect/build.gradle.kts @@ -21,7 +21,7 @@ repositories { maven(url = "https://maven.pkg.jetbrains.space/public/p/ktor/eap") } -val idea = "222.4459.20" +val idea = "222.4459.23" grammarKit { intellijRelease.set(idea) From 9441635469d972ff3a5f81e6d1fb1ca3c8c117de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Nov 2022 14:20:41 +0000 Subject: [PATCH 6/8] Bump idea from 222.4459.23 to 222.4459.24 (#108) Bumps `idea` from 222.4459.23 to 222.4459.24. Updates `core-impl` from 222.4459.23 to 222.4459.24 Updates `util-ui` from 222.4459.23 to 222.4459.24 Updates `project-model-impl` from 222.4459.23 to 222.4459.24 Updates `analysis-impl` from 222.4459.23 to 222.4459.24 --- updated-dependencies: - dependency-name: com.jetbrains.intellij.platform:core-impl dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: com.jetbrains.intellij.platform:util-ui dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: com.jetbrains.intellij.platform:project-model-impl dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: com.jetbrains.intellij.platform:analysis-impl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- postgres-native-sqldelight-dialect/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-native-sqldelight-dialect/build.gradle.kts b/postgres-native-sqldelight-dialect/build.gradle.kts index b24a64e..14fbcd2 100644 --- a/postgres-native-sqldelight-dialect/build.gradle.kts +++ b/postgres-native-sqldelight-dialect/build.gradle.kts @@ -21,7 +21,7 @@ repositories { maven(url = "https://maven.pkg.jetbrains.space/public/p/ktor/eap") } -val idea = "222.4459.23" +val idea = "222.4459.24" grammarKit { intellijRelease.set(idea) From 28e2a52a25b24f1a66f1c7e01dd2a50c129f66bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Nov 2022 07:55:33 +0100 Subject: [PATCH 7/8] Bump multiplatform from 1.7.21 to 1.7.22 (#109) --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index a58fe35..906184a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -3,7 +3,7 @@ import org.jetbrains.kotlin.gradle.dsl.* import java.util.* plugins { - kotlin("multiplatform") version "1.7.21" apply false + kotlin("multiplatform") version "1.7.22" apply false `maven-publish` signing id("io.github.gradle-nexus.publish-plugin") version "1.1.0" From 9760a0dd513106671ef580b992447cf4d0a89c73 Mon Sep 17 00:00:00 2001 From: Philip Wedemann <22521688+hfhbd@users.noreply.github.com> Date: Sun, 4 Dec 2022 14:35:37 +0100 Subject: [PATCH 8/8] Fix row number -1 is out of range 0..-1 (#113) Co-authored-by: hfhbd --- .../sqldelight/postgresdriver/PostgresNativeDriver.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/PostgresNativeDriver.kt b/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/PostgresNativeDriver.kt index 629e995..af9ae35 100644 --- a/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/PostgresNativeDriver.kt +++ b/postgres-native-sqldelight-driver/src/commonMain/kotlin/app/softwork/sqldelight/postgresdriver/PostgresNativeDriver.kt @@ -166,8 +166,9 @@ public class PostgresNativeDriver( executeQuery(null, "SELECT name FROM pg_prepared_statements WHERE name = $1", parameters = 1, binders = { bindString(0, identifier.toString()) }, mapper = { - it.next() - it.getString(0) + if (it.next()) { + it.getString(0) + } else null }) return result.value != null }