diff --git a/README.md b/README.md index 63a23e6..f581747 100644 --- a/README.md +++ b/README.md @@ -144,12 +144,9 @@ Starts the synchronization to and from remote. Multiple browser windows are supp - with sync direction - prefix `to:` to sync only from local to remote, example: `to:tasks` - prefix `from:` to sync only from remote to local, example: `from:tasks` - - with result reduction, effects only the remote to local sync - - [filter](https://github.com/mevdschee/php-crud-api?tab=readme-ov-file#filters), example: `tasks?filter=done,eq,0` - - [column selection](https://github.com/mevdschee/php-crud-api?tab=readme-ov-file#column-selection), example: `tasks?include=id,title` - - [other ...](https://github.com/mevdschee/php-crud-api?tab=readme-ov-file#list) - `options`: `` *optional* - `interval`: ``, default `1000` milliseconds + - `batchSize`: ``, default `10` documents, decrease to avoid memory issues with large documents A local table can be synchronized with only one remote table. diff --git a/demo-app/src/store.js b/demo-app/src/store.js index c075609..3bde0eb 100644 --- a/demo-app/src/store.js +++ b/demo-app/src/store.js @@ -10,7 +10,7 @@ import Sync from 'dexie-mysql-sync' // Setup the local database // Adding $created and $deleted as index allows to query on these fields const db = new Dexie('databaseName') -db.version(1).stores({ +db.version(2).stores({ tasks: '++id, title, done, $created, $deleted', files: '++id, name, type, size, $created, $deleted', }) @@ -18,7 +18,7 @@ db.version(1).stores({ // Start the synchronization const sync = new Sync() sync.add(db.tasks, 'tasks') -sync.add(db.files, 'files') +sync.add(db.files, 'files', { batchSize: 1 }) // Export the database and sync objects export { db, sync, useLiveQuery } \ No newline at end of file diff --git a/flowchart-sync.drawio b/flowchart-sync.drawio index 3c08daa..5e0bc04 100644 --- a/flowchart-sync.drawio +++ b/flowchart-sync.drawio @@ -1,6 +1,6 @@ - + - + @@ -20,7 +20,7 @@ - + @@ -38,7 +38,7 @@ - + @@ -51,28 +51,25 @@ - - + + - + - + - - - - + @@ -115,20 +112,20 @@ - - + + - + - + - + @@ -138,7 +135,7 @@ - + @@ -149,7 +146,7 @@ - + @@ -182,6 +179,18 @@ + + + + + + + + + + + + diff --git a/flowchart-sync.webp b/flowchart-sync.webp index da10235..e7bf272 100644 Binary files a/flowchart-sync.webp and b/flowchart-sync.webp differ diff --git a/lib/add.js b/lib/add.js index 8eeffbc..a9d6115 100644 --- a/lib/add.js +++ b/lib/add.js @@ -7,9 +7,13 @@ export default async function add(table, path, options = {}) { const api = this.api const debug = this.debug + const batchSize = options.batchSize || 10 const localStorageKey = `dexie-mysql-sync > ${table.name} > ${path}` debug('sync()', { table, path, options: { ...options }}) + + // Avoid that filters are passed with the path + if (path.indexOf('?') !== -1) throw new Error('The path must not contain "?".') // Avoid that the same table is synchronized twice if (this.syncs.filter(s => s.table.name === table.name).length) throw new Error('The same Dexie.js table cannot be synchronized twice.') @@ -49,10 +53,9 @@ export default async function add(table, path, options = {}) { } }) - // Split the path to table and filter options - const pathRegEx = /^(to:|from:)?(.*?)(\?.*)?$/ + // Extract the remote table from the path + const pathRegEx = /^(to:|from:)?(.+)$/ const remoteTable = path.replace(pathRegEx, '$2') - const filter = path.replace(pathRegEx, '$3').replace('include=', 'include=id,$created,$updated,$deleted') // Run synchronization with interval async function runSync() { @@ -62,12 +65,12 @@ export default async function add(table, path, options = {}) { // Sync from remote if (!path.startsWith('to:')) { - await syncFromRemote(table, path, api, remoteTable, filter, localStorageKey) + await syncFromRemote(table, path, api, remoteTable, localStorageKey, batchSize) } // Sync to remote if (!path.startsWith('from:')) { - await syncToRemote(table, api, remoteTable) + await syncToRemote(table, api, remoteTable, batchSize) } // User is offline diff --git a/lib/syncFromRemote.js b/lib/syncFromRemote.js index 619388e..eb507fa 100644 --- a/lib/syncFromRemote.js +++ b/lib/syncFromRemote.js @@ -1,7 +1,7 @@ import debug from './debug' -export default async function syncFromRemote(table, path, api, remoteTable, filter, localStorageKey) { - //debug('sync from remote > start', { table, path, api, remoteTable, filter, localStorageKey }) +export default async function syncFromRemote(table, path, api, remoteTable, localStorageKey, batchSize) { + //debug('sync from remote > start', { table, path, api, remoteTable, localStorageKey }) // Catch errors try { @@ -12,59 +12,79 @@ export default async function syncFromRemote(table, path, api, remoteTable, filt // Get last sync time const lastSyncTime = parseInt(window.localStorage.getItem(localStorageKey) || 0) - // Get not synchronized remote docs - const lastSyncFilter = (filter.indexOf('?') === -1 ? '?' : '&') + `filter=$synchronized,ge,${lastSyncTime}` - const remoteDocs = (await api.list(remoteTable + filter + lastSyncFilter)).records + // Run in batches + let syncCompleted = false + let pageNumber = 0 + while (!syncCompleted) { - // Found not synchronized remote docs - if (remoteDocs.length) { + // Get not synchronized remote docs + pageNumber++ + const filter = `$synchronized,ge,${lastSyncTime}` + const include = `id,$updated,$deleted` + const order = `$synchronized` + const page = `${pageNumber},${batchSize}` + const remoteDocs = (await api.list(remoteTable, { filter, include, order, page })).records - // Loop remote docs - for (const remoteDoc of remoteDocs) { + // Found not synchronized remote docs + if (remoteDocs.length) { - // Find local doc - const localDoc = await table.get(remoteDoc.id) + // Loop remote docs + for (let remoteDoc of remoteDocs) { - // Local doc exists - if (localDoc) { + // Find local doc + const localDoc = await table.get(remoteDoc.id) - // Remote doc is newer - if (remoteDoc.$updated > localDoc.$updated) { + // Local doc exists + if (localDoc) { - // Define updated local doc - const updatedLocalDoc = { ...remoteDoc, $synchronized: Date.now() } + // Remote doc is newer + if (remoteDoc.$updated > localDoc.$updated) { - // Update local doc - await table.update(localDoc.id, updatedLocalDoc) - debug('sync from remote > update doc', updatedLocalDoc) + // Get full remote doc + remoteDoc = await api.read(remoteTable, remoteDoc.id) - // Deletion flag set - if (remoteDoc.$deleted) { + // Define updated local doc + const updatedLocalDoc = { ...remoteDoc, $synchronized: Date.now() } - // Delete local doc - await table.delete(remoteDoc.id) + // Update local doc + await table.update(localDoc.id, updatedLocalDoc) + debug('sync from remote > update doc', updatedLocalDoc) + + // Deletion flag set + if (remoteDoc.$deleted) { + + // Delete local doc + await table.delete(remoteDoc.id) + + } } - } + // Local doc does not exist + } else { + + // No deletion flag + if (!remoteDoc.$deleted) { - // Local doc does not exist - } else { - - // No deletion flag - if (!remoteDoc.$deleted) { + // Get full remote doc + remoteDoc = await api.read(remoteTable, remoteDoc.id) - // Define new local doc - const newLocalDoc = { ...remoteDoc, $synchronized: Date.now() } + // Define new local doc + const newLocalDoc = { ...remoteDoc, $synchronized: Date.now() } - // Create local doc - await table.add(newLocalDoc) - debug('sync from remote > create doc', newLocalDoc) + // Create local doc + await table.add(newLocalDoc) + debug('sync from remote > create doc', newLocalDoc) + + } - } + } - } + } + // Not found not synchronized remote docs + } else { + syncCompleted = true } } diff --git a/lib/syncToRemote.js b/lib/syncToRemote.js index 506cf6d..6d10787 100644 --- a/lib/syncToRemote.js +++ b/lib/syncToRemote.js @@ -1,77 +1,89 @@ import debug from './debug' -export default async function syncToRemote(table, api, remoteTable) { +export default async function syncToRemote(table, api, remoteTable, batchSize) { //debug('sync to remote > start', { table, api, remoteTable }) // Catch errors try { - // Get not synchronized local docs - const localDocs = (await table.filter(doc => !doc.$synchronized).toArray()) + // Run in batches + let syncCompleted = false + while (!syncCompleted) { - // Found not synchronized local docs - if (localDocs.length) { + // Get not synchronized local docs + const localDocs = (await table.filter(doc => !doc.$synchronized).limit(batchSize).toArray()) - // Get relevant remote docs - const localDocIdStr = localDocs.map(doc => doc.id).join(',') - const remoteDocs = (await api.list(remoteTable, { filter: `id,in,${localDocIdStr}` })).records + // Found not synchronized local docs + if (localDocs.length) { - // Loop local docs - for (const localDoc of localDocs) { + // Get relevant remote docs + const localDocIdStr = localDocs.map(doc => doc.id).join(',') + const filter = `id,in,${localDocIdStr}` + const include = 'id,$updated' + const remoteDocs = (await api.list(remoteTable, { filter, include })).records - // Find remote doc - const remoteDoc = remoteDocs.filter(doc => doc.id === localDoc.id)[0] + // Loop local docs + for (const localDoc of localDocs) { - // Remote doc exists - if (remoteDoc) { + // Find remote doc + const remoteDoc = remoteDocs.filter(doc => doc.id === localDoc.id)[0] - // Local doc is newer - if (localDoc.$updated > remoteDoc.$updated) { + // Remote doc exists + if (remoteDoc) { - // Define updated remote doc - const updatedRemoteDoc = { ...localDoc, $synchronized: Date.now() } + // Local doc is newer + if (localDoc.$updated > remoteDoc.$updated) { - // Update remote doc - await api.update(remoteTable, remoteDoc.id, updatedRemoteDoc) - debug('sync to remote > update doc', updatedRemoteDoc) + // Define updated remote doc + const updatedRemoteDoc = { ...localDoc, $synchronized: Date.now() } - // Update synchronized date after update to ensure recognation for other syncs - await api.update(remoteTable, remoteDoc.id, { $synchronized: Date.now() }) + // Update remote doc + await api.update(remoteTable, remoteDoc.id, updatedRemoteDoc) + debug('sync to remote > update doc', updatedRemoteDoc) - } + // Update synchronized date after update to ensure recognation for other syncs + await api.update(remoteTable, remoteDoc.id, { $synchronized: Date.now() }) - // Remote doc does not exist - } else { + } - // Define new remote doc - const newRemoteDoc = { ...localDoc, $synchronized: Date.now() } + // Remote doc does not exist + } else { - // Create remote doc - await api.create(remoteTable, newRemoteDoc) - debug('sync to remote > create doc', newRemoteDoc) + // Define new remote doc + const newRemoteDoc = { ...localDoc, $synchronized: Date.now() } - // Update synchronized date after creation to ensure recognation for other syncs - await api.update(remoteTable, localDoc.id, { $synchronized: Date.now() }) + // Create remote doc + await api.create(remoteTable, newRemoteDoc) + debug('sync to remote > create doc', newRemoteDoc) - } + // Update synchronized date after creation to ensure recognation for other syncs + await api.update(remoteTable, localDoc.id, { $synchronized: Date.now() }) + + } + + // Deletion flag is set + if (localDoc.$deleted) { - // Deletion flag is set - if (localDoc.$deleted) { + // Delete the local doc + await table.delete(localDoc.id) - // Delete the local doc - await table.delete(localDoc.id) + // Deletion flag is not set + } else { - // Deletion flag is not set - } else { + // Define the updated local doc (workaround to keep the current $updated value) + const updatedLocalDoc = { $updated: 'keep', $synchronized: Date.now() } - // Define the updated local doc (workaround to keep the current $updated value) - const updatedLocalDoc = { $updated: 'keep', $synchronized: Date.now() } + // Update the local doc + await table.update(localDoc.id, updatedLocalDoc) + + } - // Update the local doc - await table.update(localDoc.id, updatedLocalDoc) - } + + // Not found not synchronized local docs + } else { + syncCompleted = true } } diff --git a/package-lock.json b/package-lock.json index 4d1e99b..f5ca6e4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "dexie-mysql-sync", - "version": "5.1.0", + "version": "6.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dexie-mysql-sync", - "version": "5.1.0", + "version": "6.0.0", "license": "MIT", "dependencies": { "js-crud-api": "^0.4.0", diff --git a/package.json b/package.json index a7abc80..7fbe4c9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dexie-mysql-sync", - "version": "5.1.0", + "version": "6.0.0", "description": "", "main": "lib/index.js", "type": "module",