From 1341249254cb0d2e708ae2a1da27d9cd74863d28 Mon Sep 17 00:00:00 2001
From: Stephen Lombardo <sjlombardo@zetetic.net>
Date: Thu, 9 Feb 2023 08:12:45 -0500
Subject: [PATCH 1/2] upstream fix for problem where optimizing an fts5 table
 can causes it to become unreadable (35bed981,459d986d38fc)

---
 ext/fts5/fts5_index.c            |  6 ++-
 ext/fts5/test/fts5optimize2.test | 64 ++++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+), 2 deletions(-)
 create mode 100644 ext/fts5/test/fts5optimize2.test

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index d2ec9ad5a..6afe30e05 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -54,6 +54,8 @@
 # error "FTS5_MAX_PREFIX_INDEXES is too large"
 #endif
 
+#define FTS5_MAX_LEVEL 64
+
 /*
 ** Details:
 **
@@ -4766,10 +4768,10 @@ static Fts5Structure *fts5IndexOptimizeStruct(
   if( pNew ){
     Fts5StructureLevel *pLvl;
     nByte = nSeg * sizeof(Fts5StructureSegment);
-    pNew->nLevel = pStruct->nLevel+1;
+    pNew->nLevel = MIN(pStruct->nLevel+1, FTS5_MAX_LEVEL);
     pNew->nRef = 1;
     pNew->nWriteCounter = pStruct->nWriteCounter;
-    pLvl = &pNew->aLevel[pStruct->nLevel];
+    pLvl = &pNew->aLevel[pNew->nLevel-1];
     pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
     if( pLvl->aSeg ){
       int iLvl, iSeg;
diff --git a/ext/fts5/test/fts5optimize2.test b/ext/fts5/test/fts5optimize2.test
new file mode 100644
index 000000000..a0782ee79
--- /dev/null
+++ b/ext/fts5/test/fts5optimize2.test
@@ -0,0 +1,64 @@
+# 2014 Dec 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# TESTRUNNER: slow
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5optimize2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+set nLoop 2500
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+}
+
+do_test 1.1 {
+  for {set ii 0} {$ii < $nLoop} {incr ii} {
+    execsql {
+      INSERT INTO t1 VALUES('abc def ghi');
+      INSERT INTO t1 VALUES('jkl mno pqr');
+      INSERT INTO t1(t1) VALUES('optimize');
+    }
+  }
+} {}
+
+do_execsql_test 1.2 {
+  SELECT count(*) FROM t1('mno')
+} $nLoop
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(x);
+  INSERT INTO t2(t2, rank) VALUES('pgsz', 32);
+}
+
+do_test 2.1 {
+  for {set ii 0} {$ii < $nLoop} {incr ii} {
+    execsql {
+      INSERT INTO t2 VALUES('abc def ghi');
+      INSERT INTO t2 VALUES('jkl mno pqr');
+      INSERT INTO t2(t2, rank) VALUES('merge', -1);
+    }
+  }
+} {}
+
+do_execsql_test 2.2 {
+  SELECT count(*) FROM t2('mno')
+} $nLoop
+
+finish_test

From cb4a5a6bda872eec245f4cac8d3665273665c6d4 Mon Sep 17 00:00:00 2001
From: Stephen Lombardo <sjlombardo@zetetic.net>
Date: Wed, 20 Sep 2023 15:59:14 -0400
Subject: [PATCH 2/2] upstream fix for fts5 secure-delete corruption
 (cc0f82a480a400c6)

---
 ext/fts5/fts5_index.c         | 25 +++++++++++--------------
 ext/fts5/test/fts5secure.test | 13 +++++++++++++
 2 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index eaeeeff4f..d4fee74b0 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -4711,7 +4711,6 @@ static void fts5DoSecureDelete(
   int iIdx = 0;
   int iStart = 0;
   int iKeyOff = 0;
-  int iPrevKeyOff = 0;
   int iDelKeyOff = 0;       /* Offset of deleted key, if any */
 
   nIdx = nPg-iPgIdx;
@@ -4893,26 +4892,24 @@ static void fts5DoSecureDelete(
     }
 
     if( p->rc==SQLITE_OK ){
-      const int nMove = nPg - iNextOff;
-      int nShift = 0;
+      const int nMove = nPg - iNextOff;     /* Number of bytes to move */
+      int nShift = iNextOff - iOff;         /* Distance to move them */
+      int iPrevKeyOut = 0;
+      int iKeyIn = 0;
 
       memmove(&aPg[iOff], &aPg[iNextOff], nMove);
-      iPgIdx -= (iNextOff - iOff);
+      iPgIdx -= nShift;
       nPg = iPgIdx;
       fts5PutU16(&aPg[2], iPgIdx);
 
-      nShift = iNextOff - iOff;
-      for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdx<nIdx; /* no-op */){
+      for(iIdx=0; iIdx<nIdx; /* no-op */){
         u32 iVal = 0;
         iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
-        iKeyOff += iVal;
-        if( iKeyOff!=iDelKeyOff ){
-          if( iKeyOff>iOff ){
-            iKeyOff -= nShift;
-            nShift = 0;
-          }
-          nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff);
-          iPrevKeyOff = iKeyOff;
+        iKeyIn += iVal;
+        if( iKeyIn!=iDelKeyOff ){
+          int iKeyOut = (iKeyIn - (iKeyIn>iOff ? nShift : 0));
+          nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOut - iPrevKeyOut);
+          iPrevKeyOut = iKeyOut;
         }
       }
 
diff --git a/ext/fts5/test/fts5secure.test b/ext/fts5/test/fts5secure.test
index 50d84cef7..7e8aea3ba 100644
--- a/ext/fts5/test/fts5secure.test
+++ b/ext/fts5/test/fts5secure.test
@@ -273,6 +273,19 @@ do_execsql_test 5.3 {
 do_execsql_test 5.4 { SELECT rowid FROM t1('abc'); } 2
 do_execsql_test 5.5 { SELECT rowid FROM t1('aa'); } 2
 
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE fts USING fts5(content);
+  INSERT INTO fts(fts, rank) VALUES ('secure-delete', 1);
+  INSERT INTO fts(rowid, content) VALUES
+    (3407, 'profile profile profile profile profile profile profile profile pull pulling pulling really');
+  DELETE FROM fts WHERE rowid IS 3407;
+  INSERT INTO fts(fts) VALUES ('integrity-check');
+}
+
+
 
 finish_test