From 8cb25d9026cade979d2a05e5625ea176a713d14e Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 14 May 2025 13:01:31 +0400 Subject: [PATCH 1/5] add LIBSQL_CHECKPOINT_ONLY_FULL compile option --- libsql-sqlite3/src/wal.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/libsql-sqlite3/src/wal.c b/libsql-sqlite3/src/wal.c index cbf11a6d00..7d0f2a18b8 100644 --- a/libsql-sqlite3/src/wal.c +++ b/libsql-sqlite3/src/wal.c @@ -2066,6 +2066,15 @@ static int walCheckpoint( } } + +#ifdef LIBSQL_CHECKPOINT_ONLY_FULL + // in case of LIBSQL_CHECKPOINT_ONLY_FULL option we want to either checkpoint whole WAL or quickly abort the checkpoint + if( mxSafeFrame!=walIndexHdr(pWal)->mxFrame ){ + rc = SQLITE_BUSY; + goto walcheckpoint_out; + } +#endif + /* Allocate the iterator */ if( pInfo->nBackfillnBackfill, &pIter, mxSafeFrame, xCb == NULL); @@ -2130,18 +2139,10 @@ static int walCheckpoint( /* If work was actually accomplished... */ if( rc==SQLITE_OK ){ -#ifdef LIBSQL_CHECKPOINT_CALLBACK_ON_ANY_FRAME_WRITTEN - if (xCb) { - rc = (xCb)(pCbData, mxSafeFrame, NULL, 0, 0, 0); - } -#endif - if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ -#ifndef LIBSQL_CHECKPOINT_CALLBACK_ON_ANY_FRAME_WRITTEN if (xCb) { rc = (xCb)(pCbData, mxSafeFrame, NULL, 0, 0, 0); } -#endif if( rc==SQLITE_OK ){ i64 szDb = pWal->hdr.nPage*(i64)szPage; testcase( IS_BIG_INT(szDb) ); From 24e6124ed58539bcdebfd26a285a9e5002ad23b7 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 14 May 2025 13:04:16 +0400 Subject: [PATCH 2/5] build bundles --- libsql-ffi/bundled/bindings/bindgen.rs | 19 ++++++++++++++----- libsql-ffi/bundled/src/sqlite3.c | 17 +++++++++-------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/libsql-ffi/bundled/bindings/bindgen.rs b/libsql-ffi/bundled/bindings/bindgen.rs index e0a1e9bb43..343361ba38 100644 --- a/libsql-ffi/bundled/bindings/bindgen.rs +++ b/libsql-ffi/bundled/bindings/bindgen.rs @@ -23,6 +23,7 @@ extern "C" { ) -> ::std::os::raw::c_int; } +pub const __GNUC_VA_LIST: i32 = 1; pub const SQLITE_VERSION: &[u8; 7] = b"3.45.1\0"; pub const SQLITE_VERSION_NUMBER: i32 = 3045001; pub const SQLITE_SOURCE_ID: &[u8; 85] = @@ -501,8 +502,8 @@ pub const FTS5_TOKENIZE_DOCUMENT: i32 = 4; pub const FTS5_TOKENIZE_AUX: i32 = 8; pub const FTS5_TOKEN_COLOCATED: i32 = 1; pub const WAL_SAVEPOINT_NDATA: i32 = 4; -pub type __gnuc_va_list = __builtin_va_list; pub type va_list = __builtin_va_list; +pub type __gnuc_va_list = __builtin_va_list; extern "C" { pub static sqlite3_version: [::std::os::raw::c_char; 0usize]; } @@ -939,7 +940,7 @@ extern "C" { extern "C" { pub fn sqlite3_vmprintf( arg1: *const ::std::os::raw::c_char, - arg2: va_list, + arg2: *mut __va_list_tag, ) -> *mut ::std::os::raw::c_char; } extern "C" { @@ -955,7 +956,7 @@ extern "C" { arg1: ::std::os::raw::c_int, arg2: *mut ::std::os::raw::c_char, arg3: *const ::std::os::raw::c_char, - arg4: va_list, + arg4: *mut __va_list_tag, ) -> *mut ::std::os::raw::c_char; } extern "C" { @@ -2505,7 +2506,7 @@ extern "C" { pub fn sqlite3_str_vappendf( arg1: *mut sqlite3_str, zFormat: *const ::std::os::raw::c_char, - arg2: va_list, + arg2: *mut __va_list_tag, ); } extern "C" { @@ -3573,4 +3574,12 @@ extern "C" { extern "C" { pub static sqlite3_wal_manager: libsql_wal_manager; } -pub type __builtin_va_list = *mut ::std::os::raw::c_char; +pub type __builtin_va_list = [__va_list_tag; 1usize]; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct __va_list_tag { + pub gp_offset: ::std::os::raw::c_uint, + pub fp_offset: ::std::os::raw::c_uint, + pub overflow_arg_area: *mut ::std::os::raw::c_void, + pub reg_save_area: *mut ::std::os::raw::c_void, +} diff --git a/libsql-ffi/bundled/src/sqlite3.c b/libsql-ffi/bundled/src/sqlite3.c index 6f8c31233b..265c94ad12 100644 --- a/libsql-ffi/bundled/src/sqlite3.c +++ b/libsql-ffi/bundled/src/sqlite3.c @@ -67613,6 +67613,15 @@ static int walCheckpoint( } } + +#ifdef LIBSQL_CHECKPOINT_ONLY_FULL + // in case of LIBSQL_CHECKPOINT_ONLY_FULL option we want to either checkpoint whole WAL or quickly abort the checkpoint + if( mxSafeFrame!=walIndexHdr(pWal)->mxFrame ){ + rc = SQLITE_BUSY; + goto walcheckpoint_out; + } +#endif + /* Allocate the iterator */ if( pInfo->nBackfillnBackfill, &pIter, mxSafeFrame, xCb == NULL); @@ -67677,18 +67686,10 @@ static int walCheckpoint( /* If work was actually accomplished... */ if( rc==SQLITE_OK ){ -#ifdef LIBSQL_CHECKPOINT_CALLBACK_ON_ANY_FRAME_WRITTEN - if (xCb) { - rc = (xCb)(pCbData, mxSafeFrame, NULL, 0, 0, 0); - } -#endif - if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ -#ifndef LIBSQL_CHECKPOINT_CALLBACK_ON_ANY_FRAME_WRITTEN if (xCb) { rc = (xCb)(pCbData, mxSafeFrame, NULL, 0, 0, 0); } -#endif if( rc==SQLITE_OK ){ i64 szDb = pWal->hdr.nPage*(i64)szPage; testcase( IS_BIG_INT(szDb) ); From c0e7241166421c4466380ffac7fb691930d692b6 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 14 May 2025 13:06:42 +0400 Subject: [PATCH 3/5] introduce libsql-checkpoint-only-full feature for libsql-ffi crate --- libsql-ffi/Cargo.toml | 2 +- libsql-ffi/build.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libsql-ffi/Cargo.toml b/libsql-ffi/Cargo.toml index 743f2264ae..eff5ef288d 100644 --- a/libsql-ffi/Cargo.toml +++ b/libsql-ffi/Cargo.toml @@ -48,4 +48,4 @@ sqlean-extensions = [ "sqlean-extension-text", ] libsql-disable-checkpoint-downgrade = [] -libsql-checkpoint-callback-on-any-frame-written = [] +libsql-checkpoint-only-full= [] diff --git a/libsql-ffi/build.rs b/libsql-ffi/build.rs index d48790b8f4..3de70f5e66 100644 --- a/libsql-ffi/build.rs +++ b/libsql-ffi/build.rs @@ -276,8 +276,8 @@ pub fn build_bundled(out_dir: &str, out_path: &Path) { if cfg!(feature = "libsql-disable-checkpoint-downgrade") { cfg.flag("-DLIBSQL_DISABLE_CHECKPOINT_DOWNGRADE=1"); } - if cfg!(feature = "libsql-checkpoint-callback-on-any-frame-written") { - cfg.flag("-DLIBSQL_CHECKPOINT_CALLBACK_ON_ANY_FRAME_WRITTEN=1"); + if cfg!(feature = "libsql-checkpoint-only-full") { + cfg.flag("-DLIBSQL_CHECKPOINT_ONLY_FULL=1"); } if cfg!(feature = "bundled-sqlcipher") { From fadd1387778db85c0db198b3851a5113414283f7 Mon Sep 17 00:00:00 2001 From: "Levy A." Date: Mon, 12 May 2025 19:39:14 -0300 Subject: [PATCH 4/5] feat: add `columns` information for Hrana Statements --- libsql-server/tests/standalone/mod.rs | 4 ++-- libsql/src/hrana/connection.rs | 4 ++-- libsql/src/hrana/hyper.rs | 15 ++++++++++++--- libsql/src/hrana/mod.rs | 12 +++++++++++- libsql/src/replication/connection.rs | 25 ++++++++++++++++++------- libsql/src/sync/connection.rs | 2 +- libsql/src/wasm/mod.rs | 8 ++++---- 7 files changed, 50 insertions(+), 20 deletions(-) diff --git a/libsql-server/tests/standalone/mod.rs b/libsql-server/tests/standalone/mod.rs index 0b3c631e75..046f147d4d 100644 --- a/libsql-server/tests/standalone/mod.rs +++ b/libsql-server/tests/standalone/mod.rs @@ -99,7 +99,7 @@ fn basic_metrics() { let snapshot = snapshot_metrics(); snapshot.assert_counter("libsql_server_libsql_execute_program", 3); - snapshot.assert_counter("libsql_server_user_http_response", 3); + snapshot.assert_counter("libsql_server_user_http_response", 4); for (key, (_, _, val)) in snapshot.snapshot() { if key.kind() == metrics_util::MetricKind::Counter @@ -107,7 +107,7 @@ fn basic_metrics() { { let label = key.key().labels().next().unwrap(); assert!(label.value().starts_with("libsql-remote-")); - assert_eq!(val, &metrics_util::debugging::DebugValue::Counter(3)); + assert_eq!(val, &metrics_util::debugging::DebugValue::Counter(4)); } } diff --git a/libsql/src/hrana/connection.rs b/libsql/src/hrana/connection.rs index ba8201230f..e6030ff7a5 100644 --- a/libsql/src/hrana/connection.rs +++ b/libsql/src/hrana/connection.rs @@ -80,9 +80,9 @@ where ) } - pub fn prepare(&self, sql: &str) -> crate::Result> { + pub async fn prepare(&self, sql: &str) -> crate::Result> { let stream = self.current_stream().clone(); - Statement::new(stream, sql.to_string(), true) + Statement::new(stream, sql.to_string(), true).await } } diff --git a/libsql/src/hrana/hyper.rs b/libsql/src/hrana/hyper.rs index 9ed0880af8..57840f56f4 100644 --- a/libsql/src/hrana/hyper.rs +++ b/libsql/src/hrana/hyper.rs @@ -131,7 +131,7 @@ impl Conn for HttpConnection { async fn prepare(&self, sql: &str) -> crate::Result { let stream = self.current_stream().clone(); - let stmt = crate::hrana::Statement::new(stream, sql.to_string(), true)?; + let stmt = crate::hrana::Statement::new(stream, sql.to_string(), true).await?; Ok(Statement { inner: Box::new(stmt), }) @@ -241,7 +241,16 @@ impl crate::statement::Stmt for crate::hrana::Statement { // 2. Even if we do execute query, Hrana doesn't return all info that Column exposes. // 3. Even if we would like to return some of the column info ie. column [ValueType], this information is not // present in Hrana [Col] but rather inferred from the row cell type. - vec![] + self.cols + .iter() + .map(|name| crate::Column { + name, + origin_name: None, + table_name: None, + database_name: None, + decl_type: None, + }) + .collect() } } @@ -350,7 +359,7 @@ impl Conn for HranaStream { } async fn prepare(&self, sql: &str) -> crate::Result { - let stmt = crate::hrana::Statement::new(self.clone(), sql.to_string(), true)?; + let stmt = crate::hrana::Statement::new(self.clone(), sql.to_string(), true).await?; Ok(Statement { inner: Box::new(stmt), }) diff --git a/libsql/src/hrana/mod.rs b/libsql/src/hrana/mod.rs index 2e0d6cf4ae..b29d5e4d34 100644 --- a/libsql/src/hrana/mod.rs +++ b/libsql/src/hrana/mod.rs @@ -119,13 +119,22 @@ where stream: HranaStream, close_stream: bool, inner: Stmt, + cols: Vec, } impl Statement where T: HttpSend + Send + Sync + 'static, { - pub(crate) fn new(stream: HranaStream, sql: String, want_rows: bool) -> crate::Result { + pub(crate) async fn new( + stream: HranaStream, + sql: String, + want_rows: bool, + ) -> crate::Result { + let desc = stream.describe(&sql).await?; + + let cols: Vec<_> = desc.cols.into_iter().map(|col| col.name).collect(); + // in SQLite when a multiple statements are glued together into one string, only the first one is // executed and then a handle to continue execution is returned. However Hrana API doesn't allow // passing multi-statement strings, so we just pick first one. @@ -147,6 +156,7 @@ where stream, close_stream, inner, + cols, }) } } diff --git a/libsql/src/replication/connection.rs b/libsql/src/replication/connection.rs index e92954e29d..576e6179de 100644 --- a/libsql/src/replication/connection.rs +++ b/libsql/src/replication/connection.rs @@ -1,14 +1,14 @@ // TODO(lucio): Move this to `remote/mod.rs` -use std::time::Duration; -use std::str::FromStr; -use std::sync::Arc; -use std::sync::atomic::AtomicU64; use libsql_replication::rpc::proxy::{ describe_result, query_result::RowResult, Cond, DescribeResult, ExecuteResults, NotCond, OkCond, Positional, Query, ResultRows, State as RemoteState, Step, }; use parking_lot::Mutex; +use std::str::FromStr; +use std::sync::atomic::AtomicU64; +use std::sync::Arc; +use std::time::Duration; use crate::parser; use crate::parser::StmtKind; @@ -168,7 +168,11 @@ impl From for State { } impl RemoteConnection { - pub(crate) fn new(local: LibsqlConnection, writer: Option, max_write_replication_index: Arc) -> Self { + pub(crate) fn new( + local: LibsqlConnection, + writer: Option, + max_write_replication_index: Arc, + ) -> Self { let state = Arc::new(Mutex::new(Inner::default())); Self { local, @@ -180,9 +184,16 @@ impl RemoteConnection { fn update_max_write_replication_index(&self, index: Option) { if let Some(index) = index { - let mut current = self.max_write_replication_index.load(std::sync::atomic::Ordering::SeqCst); + let mut current = self + .max_write_replication_index + .load(std::sync::atomic::Ordering::SeqCst); while index > current { - match self.max_write_replication_index.compare_exchange(current, index, std::sync::atomic::Ordering::SeqCst, std::sync::atomic::Ordering::SeqCst) { + match self.max_write_replication_index.compare_exchange( + current, + index, + std::sync::atomic::Ordering::SeqCst, + std::sync::atomic::Ordering::SeqCst, + ) { Ok(_) => break, Err(new_current) => current = new_current, } diff --git a/libsql/src/sync/connection.rs b/libsql/src/sync/connection.rs index dbd85d0cdd..807e49a2f4 100644 --- a/libsql/src/sync/connection.rs +++ b/libsql/src/sync/connection.rs @@ -128,7 +128,7 @@ impl Conn for SyncedConnection { }) } else { let stmt = Statement { - inner: Box::new(self.remote.prepare(sql)?), + inner: Box::new(self.remote.prepare(sql).await?), }; if self.read_your_writes { diff --git a/libsql/src/wasm/mod.rs b/libsql/src/wasm/mod.rs index 6345086bd5..6bcd133303 100644 --- a/libsql/src/wasm/mod.rs +++ b/libsql/src/wasm/mod.rs @@ -74,7 +74,7 @@ where self.conn.current_stream().clone(), sql.to_string(), true, - )?; + ).await?; let rows = stmt.execute(¶ms.into_params()?).await?; Ok(rows as u64) } @@ -104,7 +104,7 @@ where self.conn.current_stream().clone(), sql.to_string(), true, - )?; + ).await?; let rows = stmt.query_raw(¶ms.into_params()?).await?; Ok(Rows { inner: Box::new(rows), @@ -139,7 +139,7 @@ where pub async fn query(&self, sql: &str, params: impl IntoParams) -> crate::Result { tracing::trace!("querying `{}`", sql); let stream = self.inner.stream().clone(); - let mut stmt = crate::hrana::Statement::new(stream, sql.to_string(), true)?; + let mut stmt = crate::hrana::Statement::new(stream, sql.to_string(), true).await?; let rows = stmt.query_raw(¶ms.into_params()?).await?; Ok(Rows { inner: Box::new(rows), @@ -149,7 +149,7 @@ where pub async fn execute(&self, sql: &str, params: impl IntoParams) -> crate::Result { tracing::trace!("executing `{}`", sql); let stream = self.inner.stream().clone(); - let mut stmt = crate::hrana::Statement::new(stream, sql.to_string(), true)?; + let mut stmt = crate::hrana::Statement::new(stream, sql.to_string(), true).await?; let rows = stmt.execute(¶ms.into_params()?).await?; Ok(rows as u64) } From c6ebbe1b23a5dc7d950b2a4ad33fc33c849e462f Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Wed, 14 May 2025 17:25:21 +0300 Subject: [PATCH 5/5] 0.9.7 --- Cargo.lock | 12 ++++++------ Cargo.toml | 12 ++++++------ vendored/rusqlite/Cargo.toml | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1d39a5a21c..b292723a40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2896,7 +2896,7 @@ dependencies = [ [[package]] name = "libsql" -version = "0.9.6" +version = "0.9.7" dependencies = [ "anyhow", "async-stream", @@ -2956,7 +2956,7 @@ dependencies = [ [[package]] name = "libsql-ffi" -version = "0.9.6" +version = "0.9.7" dependencies = [ "bindgen", "cc", @@ -2966,7 +2966,7 @@ dependencies = [ [[package]] name = "libsql-hrana" -version = "0.9.6" +version = "0.9.7" dependencies = [ "base64 0.21.7", "bytes", @@ -2977,7 +2977,7 @@ dependencies = [ [[package]] name = "libsql-rusqlite" -version = "0.9.6" +version = "0.9.7" dependencies = [ "bencher", "bitflags 2.6.0", @@ -3120,7 +3120,7 @@ dependencies = [ [[package]] name = "libsql-sys" -version = "0.9.6" +version = "0.9.7" dependencies = [ "bytes", "libsql-ffi", @@ -3151,7 +3151,7 @@ dependencies = [ [[package]] name = "libsql_replication" -version = "0.9.6" +version = "0.9.7" dependencies = [ "aes", "arbitrary", diff --git a/Cargo.toml b/Cargo.toml index d7177b2a72..fd0219e0d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,18 +25,18 @@ exclude = [ ] [workspace.package] -version = "0.9.6" +version = "0.9.7" authors = ["the libSQL authors"] edition = "2021" license = "MIT" repository = "https://github.com/tursodatabase/libsql" [workspace.dependencies] -libsql-ffi = { path = "libsql-ffi", version = "0.9.6" } -libsql-sys = { path = "libsql-sys", version = "0.9.6", default-features = false } -libsql-hrana = { path = "libsql-hrana", version = "0.9.6" } -libsql_replication = { path = "libsql-replication", version = "0.9.6" } -rusqlite = { package = "libsql-rusqlite", path = "vendored/rusqlite", version = "0.9.6", default-features = false, features = [ +libsql-ffi = { path = "libsql-ffi", version = "0.9.7" } +libsql-sys = { path = "libsql-sys", version = "0.9.7", default-features = false } +libsql-hrana = { path = "libsql-hrana", version = "0.9.7" } +libsql_replication = { path = "libsql-replication", version = "0.9.7" } +rusqlite = { package = "libsql-rusqlite", path = "vendored/rusqlite", version = "0.9.7", default-features = false, features = [ "libsql-experimental", "column_decltype", "load_extension", diff --git a/vendored/rusqlite/Cargo.toml b/vendored/rusqlite/Cargo.toml index 551139bdbf..9bf84936f3 100644 --- a/vendored/rusqlite/Cargo.toml +++ b/vendored/rusqlite/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "libsql-rusqlite" # Note: Update version in README.md when you change this. -version = "0.9.6" +version = "0.9.7" authors = ["The rusqlite developers"] edition = "2018" description = "Ergonomic wrapper for SQLite (libsql fork)"