From 766512bde609810e5659635c076027013e272446 Mon Sep 17 00:00:00 2001 From: Yumcc Date: Sat, 29 May 2021 12:35:55 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat:=E5=9F=BA=E4=BA=8Eredis=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E6=96=87=E6=A1=A3=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index.js | 94 ++++++++++++++++++++++++++++++++++++---------------- package.json | 6 ++-- 2 files changed, 70 insertions(+), 30 deletions(-) diff --git a/index.js b/index.js index 251efe2..a481aa4 100644 --- a/index.js +++ b/index.js @@ -1,16 +1,20 @@ var DB = require('sharedb').DB; var pg = require('pg'); +var Redis = require('ioredis'); // Postgres-backed ShareDB database function PostgresDB(options) { - if (!(this instanceof PostgresDB)) return new PostgresDB(options); - DB.call(this, options); + if (!(this instanceof PostgresDB)) return new PostgresDB(options) + DB.call(this, options) - this.closed = false; + this.closed = false + this.redis = null - this.pg_config = options; - this.pool = new pg.Pool(options); + this.pg_config = options + this.pool = new pg.Pool(options) + const redisOpts = options?.redisOpts || {} + if (Object.keys(redisOpts).length) { this.redis = new Redis(redisOpts) } }; module.exports = PostgresDB; @@ -37,10 +41,10 @@ PostgresDB.prototype.commit = function(collection, id, op, snapshot, options, ca * } * snapshot: PostgresSnapshot */ - this.pool.connect((err, client, done) => { - if (err) { + this.pool.connect((error, client, done) => { + if (error) { done(client); - callback(err); + callback(error); return; } /* @@ -59,7 +63,8 @@ PostgresDB.prototype.commit = function(collection, id, op, snapshot, options, ca * * Casting is required as postgres thinks that collection and doc_id are * not varchar - */ + */ + const snapshotsKey = `${collection}:${id}:snapshots` const query = { name: 'sdb-commit-op-and-snap', text: `WITH snapshot_id AS ( @@ -93,7 +98,7 @@ WHERE ( ) ) AND EXISTS (SELECT 1 FROM snapshot_id) RETURNING version`, - values: [collection,id,snapshot.v, snapshot.type, snapshot.data,op] + values: [ collection, id, snapshot.v, snapshot.type, snapshot.data, op ] } client.query(query, (err, res) => { if (err) { @@ -103,51 +108,75 @@ RETURNING version`, callback(null,false) } else { + RedisSetKey(snapshotsKey, + { + id, + v: snapshot.v, + type: snapshot.type, + data: snapshot.data + }, + this.redis) done(client); callback(null,true) } }) - + }) }; // Get the named document from the database. The callback is called with (err, // snapshot). A snapshot with a version of zero is returned if the docuemnt // has never been created in the database. -PostgresDB.prototype.getSnapshot = function(collection, id, fields, options, callback) { - this.pool.connect(function(err, client, done) { - if (err) { +PostgresDB.prototype.getSnapshot = async function (collection, id, fields, options, callback) { + const snapshotsKey = `${collection}:${id}:snapshots` + if (this.redis) { + const isExists = await this.redis.exists(snapshotsKey) + if (isExists) { + try { + const snapshot = await this.redis.get(snapshotsKey) + callback(null, JSON.parse(snapshot)) + return + } catch (error) { + console.log(error) + } + } + } + + this.pool.connect((error, client, done) => { + if (error) { done(client); - callback(err); + callback(error); return; } client.query( 'SELECT version, data, doc_type FROM snapshots WHERE collection = $1 AND doc_id = $2 LIMIT 1', [collection, id], - function(err, res) { + (err, res) => { done(); if (err) { callback(err); return; } if (res.rows.length) { - var row = res.rows[0] - var snapshot = new PostgresSnapshot( + let row = res.rows[0] + let snapshot = new PostgresSnapshot( id, row.version, row.doc_type, row.data, undefined // TODO: metadata ) + RedisSetKey(snapshotsKey, snapshot, this.redis) callback(null, snapshot); } else { - var snapshot = new PostgresSnapshot( + let snapshot = new PostgresSnapshot( id, 0, null, undefined, undefined ) + RedisSetKey(snapshotsKey, snapshot, this.redis) callback(null, snapshot); } } @@ -164,28 +193,29 @@ PostgresDB.prototype.getSnapshot = function(collection, id, fields, options, cal // The version will be inferred from the parameters if it is missing. // // Callback should be called as callback(error, [list of ops]); -PostgresDB.prototype.getOps = function(collection, id, from, to, options, callback) { - this.pool.connect(function(err, client, done) { - if (err) { +PostgresDB.prototype.getOps = async function (collection, id, from, to, options, callback) { + this.pool.connect((error, client, done) => { + if (error) { done(client); - callback(err); + callback(error); return; } - var cmd = 'SELECT version, operation FROM ops WHERE collection = $1 AND doc_id = $2 AND version > $3 '; - var params = [collection, id, from]; + let cmd = 'SELECT version, operation FROM ops WHERE collection = $1 AND doc_id = $2 AND version > $3 '; + let params = [collection, id, from]; if(to || to == 0) { cmd += ' AND version <= $4'; params.push(to)} cmd += ' order by version'; client.query( cmd, params, - function(err, res) { + (err, res) => { done(); if (err) { callback(err); return; } - callback(null, res.rows.map(function(row) { + const arr = res.rows.map(function(row) { return row.operation; - })); + }) + callback(null, arr) } ) }) @@ -198,3 +228,11 @@ function PostgresSnapshot(id, version, type, data, meta) { this.data = data; this.m = meta; } + +function RedisSetKey (key, data, redis) { + if (!redis) { + return + } + + redis.set(key, JSON.stringify(data)) +} \ No newline at end of file diff --git a/package.json b/package.json index 051bc07..d6351c2 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "sharedb", "sharejs", "share", - "postgres" + "postgres", + "redis" ], "repository": { "type": "git", @@ -21,6 +22,7 @@ "dependencies": { "pg": "^8.5.1", "pg-pool": "^3.2.1", - "sharedb": "^1.6.0" + "sharedb": "^1.6.0", + "ioredis": "^4.27.3" } } From 8caaa66e670dc8a4c922dd995310557fb486e809 Mon Sep 17 00:00:00 2001 From: Yumcc Date: Sat, 29 May 2021 13:03:57 +0800 Subject: [PATCH 2/3] =?UTF-8?q?doc:=E4=BF=AE=E6=94=B9package.json=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 6 ++++++ package.json | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1adfb00..2b780cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## Note about re-versioning + +感谢原先的 `sharedb-postgres` 和 `@plotdb/sharedb-postgre`，由于读操作是直接从数据库获取，业务庞大时对数据库并不友好，所以我这里基于redis加多一层缓存，具体可查看文档，维护在`@yumcc/sharedb-postgre`. +Thanks to the original `sharedb-postgres` and `@plotdb/sharedb-postgre`, since the read operation is obtained directly from the database, it is not friendly to the database when the business is large, so I add an extra layer of cache based on redis here, which can be viewed for details Documentation, maintained in `@yumcc/sharedb-postgre`. + + ## 4.0.1 - upgrade sharedb diff --git a/package.json b/package.json index d6351c2..985c742 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { - "name": "@plotdb/sharedb-postgres", - "version": "4.0.1", - "description": "PostgreSQL adapter for ShareDB. forked from share/sharedb-postgres", + "name": "@yumcc/sharedb-postgres", + "version": "5.0.0", + "description": "PostgreSQL adapter for ShareDB. forked from share/sharedb-postgres.Use redis for data caching.", "main": "index.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1" @@ -17,7 +17,7 @@ ], "repository": { "type": "git", - "url": "https://github.com/plotdb/sharedb-postgres" + "url": "https://github.com/yumcc-com/sharedb-postgres" }, "dependencies": { "pg": "^8.5.1", From e14f24b1cf7f6dd14c4f5033b281610b305974ec Mon Sep 17 00:00:00 2001 From: Yumcc Date: Sat, 29 May 2021 13:17:43 +0800 Subject: [PATCH 3/3] =?UTF-8?q?doc:=E4=BF=AE=E6=94=B9readme=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index b73277b..3d2aca9 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,9 @@ +# @yumcc/sharedb-postgres + +感谢原先的 `sharedb-postgres` 和 `@plotdb/sharedb-postgre`，由于读操作是直接从数据库获取，业务庞大时对数据库并不友好，所以我这里基于redis加多一层缓存，具体可查看文档，维护在`@yumcc/sharedb-postgre`. + +Thanks to the original `sharedb-postgres` and `@plotdb/sharedb-postgre`, since the read operation is obtained directly from the database, it is not friendly to the database when the business is large, so I add an extra layer of cache based on redis here, which can be viewed for details Documentation, maintained in `@yumcc/sharedb-postgre`. + # @plotdb/sharedb-postgres PostgreSQL database adapter for [sharedb](https://github.com/share/sharedb). This driver can be used both as a snapshot store and oplog.