From 00fb47ac3051538162b830ae7addc1a98e89699e Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Mon, 30 Nov 2009 18:29:56 -0800 Subject: [PATCH 01/54] Fixed the demo changed .markdown to .md --- README.markdown | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 README.markdown diff --git a/README.markdown b/README.markdown deleted file mode 100644 index a404f29..0000000 --- a/README.markdown +++ /dev/null @@ -1,19 +0,0 @@ -# PostgreSQL for Javascript - -This library is a implementation of the PostgreSQL backend/frontend protocol in javascript. -It uses the node.js tcp and event libraries. A javascript md5 library is included for servers that require md5 password hashing (this is default). - -## Example use - - var sys = require("sys"); - var Postgres = require('postgres.js'); - - function onLoad() { - var db = new Postgres.Connection("database", "username", "password"); - db.query("SELECT * FROM sometable", function (data) { - sys.p(data); - }); - db.close(); - } - - From 41c6cf1ca4281cdbea9de7afb78e685f470f7377 Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Mon, 30 Nov 2009 18:34:14 -0800 Subject: [PATCH 02/54] Forgot to actually add my changes :( --- README.md | 16 ++++++++++++++++ demo.js | 8 ++++++++ 2 files changed, 24 insertions(+) create mode 100644 README.md create mode 100644 demo.js diff --git a/README.md b/README.md new file mode 100644 index 0000000..cd907a3 --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +# PostgreSQL for Javascript + +This library is a implementation of the PostgreSQL backend/frontend protocol in javascript. +It uses the node.js tcp and event libraries. A javascript md5 library is included for servers that require md5 password hashing (this is default). + +## Example use + + var sys = require("sys"); + var Postgres = require("./postgres"); + + var db = new Postgres.Connection("database", "username", "password"); + db.query("SELECT * FROM sometable", function (data) { + sys.p(data); + }); + db.close(); + diff --git a/demo.js b/demo.js new file mode 100644 index 0000000..efb9536 --- /dev/null +++ b/demo.js @@ -0,0 +1,8 @@ +var sys = require("sys"); +var Postgres = require("./postgres"); + +var db = new Postgres.Connection("database", "username", "password"); +db.query("SELECT * FROM sometable", function (data) { + sys.p(data); +}); +db.close(); From c4c344b570a8e5f84ba0f30fb80cc97474e61f08 Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Mon, 30 Nov 2009 22:20:37 -0800 Subject: [PATCH 03/54] Used ry's type-oids header to build one for javascript. * starting to write more parsing functions --- demo.js | 1 + postgres-js/bits.js | 7 +++++ postgres-js/type-oids.js | 67 ++++++++++++++++++++++++++++++++++++++++ postgres.js | 15 +++++---- 4 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 postgres-js/type-oids.js diff --git a/demo.js b/demo.js index efb9536..7b79cd8 100644 --- a/demo.js +++ b/demo.js @@ -1,5 +1,6 @@ var sys = require("sys"); var Postgres = require("./postgres"); +Postgres.DEBUG = 1; var db = new Postgres.Connection("database", "username", "password"); db.query("SELECT * FROM sometable", function (data) { diff --git a/postgres-js/bits.js b/postgres-js/bits.js index b585eeb..e8c36da 100644 --- a/postgres-js/bits.js +++ b/postgres-js/bits.js @@ -95,3 +95,10 @@ String.prototype.parse = function () { return [pos, data]; }; +exports.parseDate = function (str) { + var parts = str.split(' '); + var date = parts[0].split('-'); + var time = parts[1].split(':'); + + var timestamp = date. +} diff --git a/postgres-js/type-oids.js b/postgres-js/type-oids.js new file mode 100644 index 0000000..cf46eb7 --- /dev/null +++ b/postgres-js/type-oids.js @@ -0,0 +1,67 @@ +// taken from ry's node_postgres module + +exports.BOOL = 16 +exports.BYTEA = 17 +exports.CHAR = 18 +exports.NAME = 19 +exports.INT8 = 20 +exports.INT2 = 21 +exports.INT2VECTOR = 22 +exports.INT4 = 23 +exports.REGPROC = 24 +exports.TEXT = 25 +exports.OID = 26 +exports.TID = 27 +exports.XID = 28 +exports.CID = 29 +exports.VECTOROID = 30 +exports.PG_TYPE_RELTYPE_ = 71 +exports.PG_ATTRIBUTE_RELTYPE_ = 75 +exports.PG_PROC_RELTYPE_ = 81 +exports.PG_CLASS_RELTYPE_ = 83 +exports.POINT = 600 +exports.LSEG = 601 +exports.PATH = 602 +exports.BOX = 603 +exports.POLYGON = 604 +exports.LINE = 628 +exports.FLOAT4 = 700 +exports.FLOAT8 = 701 +exports.ABSTIME = 702 +exports.RELTIME = 703 +exports.TINTERVAL = 704 +exports.UNKNOWN = 705 +exports.CIRCLE = 718 +exports.CASH = 790 +exports.MACADDR = 829 +exports.INET = 869 +exports.CIDR = 650 +exports.INT4ARRAY = 1007 +exports.ACLITEM = 1033 +exports.BPCHAR = 1042 +exports.VARCHAR = 1043 +exports.DATE = 1082 +exports.TIME = 1083 +exports.TIMESTAMP = 1114 +exports.TIMESTAMPTZ = 1184 +exports.INTERVAL = 1186 +exports.TIMETZ = 1266 +exports.BIT = 1560 +exports.VARBIT = 1562 +exports.NUMERIC = 1700 +exports.REFCURSOR = 1790 +exports.REGPROCEDURE = 2202 +exports.REGOPER = 2203 +exports.REGOPERATOR = 2204 +exports.REGCLASS = 2205 +exports.REGTYPE = 2206 +exports.RECORD = 2249 +exports.CSTRING = 2275 +exports.ANY = 2276 +exports.ANYARRAY = 2277 +exports.VOID = 2278 +exports.TRIGGER = 2279 +exports.LANGUAGE_HANDLER = 2280 +exports.INTERNAL = 2281 +exports.OPAQUE = 2282 +exports.ANYELEMENT = 2283 diff --git a/postgres.js b/postgres.js index 7488307..cfb75c0 100644 --- a/postgres.js +++ b/postgres.js @@ -5,6 +5,7 @@ process.mixin(require('./postgres-js/bits')); process.mixin(require('./postgres-js/md5')); var tcp = require("tcp"); var sys = require("sys"); +var oid = require("./postgres-js/type-oids.js"); exports.DEBUG = 0; @@ -187,14 +188,14 @@ function parse_response(code, stream) { } -exports.Connection = function (database, username, password, port) { +exports.Connection = function (database, username, password, port, host) { // Default to port 5432 if (port === undefined) { port = 5432; } - var connection = tcp.createConnection(port); + var connection = tcp.createConnection(port, host); var events = new process.EventEmitter(); var query_queue = []; var row_description; @@ -294,14 +295,16 @@ exports.Connection = function (database, username, password, port) { // TODO: investigate to see if these numbers are stable across databases or // if we need to dynamically pull them from the pg_types table switch (description.type_id) { - case 16: // bool + case oid.BOOL: value = value === 't'; break; - case 20: // int8 - case 21: // int2 - case 23: // int4 + case oid.INT8: + case oid.INT2: + case oid.INT4: value = parseInt(value, 10); break; + case oid.DATE: // date + value = parseDate(value); } } row[description.field] = value; From 04ce9a6926bcf6f9ee83860b7e72210bd452fb78 Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Wed, 2 Dec 2009 17:49:53 -0800 Subject: [PATCH 04/54] Added date parsing and date formatting --- demo.js | 1 - postgres-js/bits.js | 9 --- postgres-js/parsers.js | 77 ++++++++++++++++++++++ postgres-js/type-oids.js | 134 +++++++++++++++++++-------------------- postgres.js | 22 +++++-- 5 files changed, 162 insertions(+), 81 deletions(-) create mode 100644 postgres-js/parsers.js diff --git a/demo.js b/demo.js index 7b79cd8..efb9536 100644 --- a/demo.js +++ b/demo.js @@ -1,6 +1,5 @@ var sys = require("sys"); var Postgres = require("./postgres"); -Postgres.DEBUG = 1; var db = new Postgres.Connection("database", "username", "password"); db.query("SELECT * FROM sometable", function (data) { diff --git a/postgres-js/bits.js b/postgres-js/bits.js index 57b741f..4914bdd 100644 --- a/postgres-js/bits.js +++ b/postgres-js/bits.js @@ -143,12 +143,3 @@ }()); - - -exports.parseDate = function (str) { - var parts = str.split(' '); - var date = parts[0].split('-'); - var time = parts[1].split(':'); - - var timestamp = date. -} diff --git a/postgres-js/parsers.js b/postgres-js/parsers.js new file mode 100644 index 0000000..e1833e3 --- /dev/null +++ b/postgres-js/parsers.js @@ -0,0 +1,77 @@ +process.mixin(GLOBAL, require("sys")); +var oids = require('./type-oids'); + +// DATE parsing + +// parse a string like YYYY-MM-DD into +var date_regex = { + "ISO": /(\d{4})-(\d{2})-(\d{2}).*/, + "ISOr": "$2/$3/$1" +} +var time_regex = { + "ISO": /.*(\d{2}):(\d{2}):(\d{2}).*/, + "ISOr": "$1:$2:$3" +} +var tz_regex = { + "ISO": /.*:\d{2}-(\d{2})$/, + "ISOr": "GMT-$100" +} + +exports.parseDateFromPostgres = function (str,datestyle,OID) { + debug("parsing string: "+ str + " with datestyle: " + datestyle + " with OID: " + OID); + + var style = datestyle.split(','); + var order = style[1]; style = style[0].replace(/^\s+|\s+$/,''); + + if (!(style in date_regex)) { + sys.debug("Error datestyle not implemented: " + style); + } + + var date='',time='',tz=''; + switch(OID) { + case oids.TIMESTAMPTZ: + tz = str.replace(tz_regex[style],tz_regex[style+'r']); + debug("Timezone: " + tz); + if (tz == str) tz = ''; + case oids.TIMESTAMP: + case oids.DATE: + date = str.replace(date_regex[style],date_regex[style+'r']); + debug("date: " + date); + if (date == str) date = ''; + if (OID==oids.DATE) break; + case oids.TIME: + time = ' ' + str.replace(time_regex[style],time_regex[style+'r']); + debug("time: " + time); + if (time == str) time = ''; + } + + date = ((date=='')?'January 1, 1970':date) + ((time=='')?'':' ') + time + ((tz=='')?'':' ') + tz; + + debug("created date: " + date); + + var d = new Date(); + d.setTime(Date.parse(date)); + return d; +}; + +// turn a number 5 into a string 05 +function pad (p,num) { + num = ''+num; + return ((num.length==1)?p:'') + num; +} + +exports.formatDateForPostgres = function(d, OID) { + var date='',time='',tz=''; + switch(OID) { + case oids.TIMESTAMPTZ: + tz = '-' + d.getTimezoneOffset()/60; + case oids.TIMESTAMP: + case oids.DATE: + date = [d.getFullYear(),pad('0',d.getMonth()),pad('0',d.getDate())].join(''); + if (OID==oids.DATE) break; + case oids.TIME: + time = [pad('0',d.getHours()),pad('0',d.getMinutes()),pad('0',d.getSeconds())].join(':'); + } + + return date + ((time=='')?'':' ') + time + ((tz=='')?'':' ') + tz; +} diff --git a/postgres-js/type-oids.js b/postgres-js/type-oids.js index cf46eb7..390031e 100644 --- a/postgres-js/type-oids.js +++ b/postgres-js/type-oids.js @@ -1,67 +1,67 @@ -// taken from ry's node_postgres module - -exports.BOOL = 16 -exports.BYTEA = 17 -exports.CHAR = 18 -exports.NAME = 19 -exports.INT8 = 20 -exports.INT2 = 21 -exports.INT2VECTOR = 22 -exports.INT4 = 23 -exports.REGPROC = 24 -exports.TEXT = 25 -exports.OID = 26 -exports.TID = 27 -exports.XID = 28 -exports.CID = 29 -exports.VECTOROID = 30 -exports.PG_TYPE_RELTYPE_ = 71 -exports.PG_ATTRIBUTE_RELTYPE_ = 75 -exports.PG_PROC_RELTYPE_ = 81 -exports.PG_CLASS_RELTYPE_ = 83 -exports.POINT = 600 -exports.LSEG = 601 -exports.PATH = 602 -exports.BOX = 603 -exports.POLYGON = 604 -exports.LINE = 628 -exports.FLOAT4 = 700 -exports.FLOAT8 = 701 -exports.ABSTIME = 702 -exports.RELTIME = 703 -exports.TINTERVAL = 704 -exports.UNKNOWN = 705 -exports.CIRCLE = 718 -exports.CASH = 790 -exports.MACADDR = 829 -exports.INET = 869 -exports.CIDR = 650 -exports.INT4ARRAY = 1007 -exports.ACLITEM = 1033 -exports.BPCHAR = 1042 -exports.VARCHAR = 1043 -exports.DATE = 1082 -exports.TIME = 1083 -exports.TIMESTAMP = 1114 -exports.TIMESTAMPTZ = 1184 -exports.INTERVAL = 1186 -exports.TIMETZ = 1266 -exports.BIT = 1560 -exports.VARBIT = 1562 -exports.NUMERIC = 1700 -exports.REFCURSOR = 1790 -exports.REGPROCEDURE = 2202 -exports.REGOPER = 2203 -exports.REGOPERATOR = 2204 -exports.REGCLASS = 2205 -exports.REGTYPE = 2206 -exports.RECORD = 2249 -exports.CSTRING = 2275 -exports.ANY = 2276 -exports.ANYARRAY = 2277 -exports.VOID = 2278 -exports.TRIGGER = 2279 -exports.LANGUAGE_HANDLER = 2280 -exports.INTERNAL = 2281 -exports.OPAQUE = 2282 -exports.ANYELEMENT = 2283 +// taken from ry's node_postgres module; +; +exports.BOOL = 16; +exports.BYTEA = 17; +exports.CHAR = 18; +exports.NAME = 19; +exports.INT8 = 20; +exports.INT2 = 21; +exports.INT2VECTOR = 22; +exports.INT4 = 23; +exports.REGPROC = 24; +exports.TEXT = 25; +exports.OID = 26; +exports.TID = 27; +exports.XID = 28; +exports.CID = 29; +exports.VECTOROID = 30; +exports.PG_TYPE_RELTYPE_ = 71; +exports.PG_ATTRIBUTE_RELTYPE_ = 75; +exports.PG_PROC_RELTYPE_ = 81; +exports.PG_CLASS_RELTYPE_ = 83; +exports.POINT = 600; +exports.LSEG = 601; +exports.PATH = 602; +exports.BOX = 603; +exports.POLYGON = 604; +exports.LINE = 628; +exports.FLOAT4 = 700; +exports.FLOAT8 = 701; +exports.ABSTIME = 702; +exports.RELTIME = 703; +exports.TINTERVAL = 704; +exports.UNKNOWN = 705; +exports.CIRCLE = 718; +exports.CASH = 790; +exports.MACADDR = 829; +exports.INET = 869; +exports.CIDR = 650; +exports.INT4ARRAY = 1007; +exports.ACLITEM = 1033; +exports.BPCHAR = 1042; +exports.VARCHAR = 1043; +exports.DATE = 1082; +exports.TIME = 1083; +exports.TIMESTAMP = 1114; +exports.TIMESTAMPTZ = 1184; +exports.INTERVAL = 1186; +exports.TIMETZ = 1266; +exports.BIT = 1560; +exports.VARBIT = 1562; +exports.NUMERIC = 1700; +exports.REFCURSOR = 1790; +exports.REGPROCEDURE = 2202; +exports.REGOPER = 2203; +exports.REGOPERATOR = 2204; +exports.REGCLASS = 2205; +exports.REGTYPE = 2206; +exports.RECORD = 2249; +exports.CSTRING = 2275; +exports.ANY = 2276; +exports.ANYARRAY = 2277; +exports.VOID = 2278; +exports.TRIGGER = 2279; +exports.LANGUAGE_HANDLER = 2280; +exports.INTERNAL = 2281; +exports.OPAQUE = 2282; +exports.ANYELEMENT = 2283; diff --git a/postgres.js b/postgres.js index 65db580..e57736a 100644 --- a/postgres.js +++ b/postgres.js @@ -1,14 +1,17 @@ /*jslint bitwise: true, eqeqeq: true, immed: true, newcap: true, nomen: true, onevar: true, plusplus: true, regexp: true, undef: true, white: true, indent: 2 */ /*globals include md5 node exports */ - process.mixin(require('./postgres-js/md5')); + var bits = require('./postgres-js/bits'); +var oid = require("./postgres-js/type-oids"); +var parsers = require("./postgres-js/parsers"); var tcp = require("tcp"); var sys = require("sys"); -var oid = require("./postgres-js/type-oids.js"); exports.DEBUG = 0; +var postgres_parameters = {}; + // http://www.postgresql.org/docs/8.3/static/protocol-message-formats.html var formatter = { CopyData: function () { @@ -246,6 +249,9 @@ exports.Connection = function (database, username, password, port, host) { connection.close(); } }); + events.addListener('ParameterStatus', function(key, value) { + postgres_parameters[key] = value; + }); events.addListener('ReadyForQuery', function () { if (query_queue.length > 0) { var query = query_queue.shift(); @@ -283,8 +289,16 @@ exports.Connection = function (database, username, password, port, host) { case oid.INT4: value = parseInt(value, 10); break; - case oid.DATE: // date - value = parseDate(value); + case oid.DATE: + case oid.TIME: + case oid.TIMESTAMP: + case oid.TIMESTAMPTZ: + value = parsers.parseDateFromPostgres( + value, + postgres_parameters['DateStyle'], + description.type_id + ); + break; } } row[description.field] = value; From f8a1e8e53e2d48bf66b965ab4783359b56794f25 Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Wed, 2 Dec 2009 20:32:32 -0800 Subject: [PATCH 05/54] Changed name of libs folder. The double "postgres-js" was too annoying --- {postgres-js => lib}/bits.js | 0 {postgres-js => lib}/md5.js | 0 {postgres-js => lib}/parsers.js | 0 {postgres-js => lib}/type-oids.js | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename {postgres-js => lib}/bits.js (100%) rename {postgres-js => lib}/md5.js (100%) rename {postgres-js => lib}/parsers.js (100%) rename {postgres-js => lib}/type-oids.js (100%) diff --git a/postgres-js/bits.js b/lib/bits.js similarity index 100% rename from postgres-js/bits.js rename to lib/bits.js diff --git a/postgres-js/md5.js b/lib/md5.js similarity index 100% rename from postgres-js/md5.js rename to lib/md5.js diff --git a/postgres-js/parsers.js b/lib/parsers.js similarity index 100% rename from postgres-js/parsers.js rename to lib/parsers.js diff --git a/postgres-js/type-oids.js b/lib/type-oids.js similarity index 100% rename from postgres-js/type-oids.js rename to lib/type-oids.js From 8e6f507976c009cf30ee009c3998e36b92b7f8a8 Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Wed, 2 Dec 2009 20:33:31 -0800 Subject: [PATCH 06/54] Fixed error caused by last rename --- postgres.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/postgres.js b/postgres.js index e57736a..3e21bc9 100644 --- a/postgres.js +++ b/postgres.js @@ -1,10 +1,10 @@ /*jslint bitwise: true, eqeqeq: true, immed: true, newcap: true, nomen: true, onevar: true, plusplus: true, regexp: true, undef: true, white: true, indent: 2 */ /*globals include md5 node exports */ -process.mixin(require('./postgres-js/md5')); +process.mixin(require('./lib/md5')); -var bits = require('./postgres-js/bits'); -var oid = require("./postgres-js/type-oids"); -var parsers = require("./postgres-js/parsers"); +var bits = require('./lib/bits'); +var oid = require("./lib/type-oids"); +var parsers = require("./lib/parsers"); var tcp = require("tcp"); var sys = require("sys"); From a076b6cc9466fc30cefa535e3bc9cdfef4b2765b Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Wed, 2 Dec 2009 21:59:09 -0800 Subject: [PATCH 07/54] More changes --- .gitignore | 2 +- demo.js | 5 +++-- postgres.js | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index d2554e7..9ba7be7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ .bzr .bzrignore -test.js \ No newline at end of file +test.js diff --git a/demo.js b/demo.js index efb9536..24fdaf9 100644 --- a/demo.js +++ b/demo.js @@ -1,8 +1,9 @@ var sys = require("sys"); var Postgres = require("./postgres"); -var db = new Postgres.Connection("database", "username", "password"); -db.query("SELECT * FROM sometable", function (data) { +var db = new Postgres.Connection("dbname", "username", "password"); +db.query("SELECT * FROM test"); +db.query("SELECT * FROM test", function (data) { sys.p(data); }); db.close(); diff --git a/postgres.js b/postgres.js index 3e21bc9..e91ab7c 100644 --- a/postgres.js +++ b/postgres.js @@ -255,7 +255,7 @@ exports.Connection = function (database, username, password, port, host) { events.addListener('ReadyForQuery', function () { if (query_queue.length > 0) { var query = query_queue.shift(); - query_callback = query.callback; + query_callback = query.callback || function() {}; sendMessage('Query', [query.sql]); readyState = false; } else { From d509dc5004418bf34dc7540a8b7f4a77fcd1a937 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Wed, 9 Dec 2009 08:42:40 -0800 Subject: [PATCH 08/54] Adds parameterized queries and prepared query support to the library. As we're not using libpq, this is achieved by using dollarquoting with randomized container names to de-taint input. This is an initial attempt - there are likely bugs and oddities I haven't discovered yet. --- README.markdown | 31 +++++++++++++ postgres.js | 120 ++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 142 insertions(+), 9 deletions(-) diff --git a/README.markdown b/README.markdown index a404f29..21e4a99 100644 --- a/README.markdown +++ b/README.markdown @@ -3,6 +3,9 @@ This library is a implementation of the PostgreSQL backend/frontend protocol in javascript. It uses the node.js tcp and event libraries. A javascript md5 library is included for servers that require md5 password hashing (this is default). +This library also allows for the handling of prepared queries and parameterized queries. +As this library does not use libpq (which supports automatic parameterization), I use Postgres' dollar-quoting system to provide argument wrapping, before sending the query to the database. + ## Example use var sys = require("sys"); @@ -17,3 +20,31 @@ It uses the node.js tcp and event libraries. A javascript md5 library is includ } +## Example use of Parameterized Queries + + var sys = require("sys"); + var pg = require("postgres.js"); + + var db = new pg.Connection("database", "username", "password"); + db.query("SELECT * FROM yourtable WHERE id = ?", [1], function (data) { + + sys.p(data); + }); + db.close(); + +## Example use of Prepared Queries + + var sys = require("sys"); + var pg = require("postgres.js"); + + var db = new pg.Connection("database", "username", "password"); + + db.prepare("SELECT * FROM yourtable WHERE id = ?").addCallback( function (query) { + + sys.p(query); + query.execute(["1"], function (d) { + sys.p(d); + }); + /* More queries here. */ + }); + db.close(); \ No newline at end of file diff --git a/postgres.js b/postgres.js index f6dfca6..9fdaf35 100644 --- a/postgres.js +++ b/postgres.js @@ -167,15 +167,20 @@ function parse_response(code, stream) { } -exports.Connection = function (database, username, password, port) { +exports.Connection = function (database, username, password, host, port) { var connection, events, query_queue, row_description, query_callback, results, readyState, closeState; // Default to port 5432 if (port === undefined) { port = 5432; } + + t_host = host; + if (t_host === undefined) { + t_host = "localhost"; + } - connection = tcp.createConnection(port); + connection = tcp.createConnection(port, host=t_host); events = new process.EventEmitter(); query_queue = []; readyState = false; @@ -255,7 +260,7 @@ exports.Connection = function (database, username, password, port) { if (closeState) { connection.close(); } else { - readyState = true; + readyState = true; } } }); @@ -292,15 +297,112 @@ exports.Connection = function (database, username, password, port) { query_callback.call(this, results); }); - this.query = function (sql, callback) { - query_queue.push({sql: sql, callback: callback}); - if (readyState) { - events.emit('ReadyForQuery'); - } + this.query = function (sql, args, callback) { + + if (callback == null) { + + // This has no parameters to manipulate. + + // Assume the args value is the callback + sys.puts("Null callback"); + sys.p(args); + query_queue.push({sql: sql, callback: args || function () {} }); + if (readyState) { + events.emit('ReadyForQuery'); + } + } + else { + // We have an args list. + // This means, we have to map our ?'s and test for a variety of + // edge cases. + sys.puts("Got args."); + var i = 0; + var slice = md5(md5(sql)); + //sys.p(slice); + var offset = Math.floor(Math.random() * 10); + cont = "$" + slice.replace(/\d/g, "").slice(offset,4+offset) + "$"; + var treated = sql; + sys.p(cont); + if (sql.match(/\?/)) { + treated = sql.replace(/\?/g, function (str, offset, s) { + if (!args[i]) { + // raise an error + throw new Error("Argument "+i+" does not exist!"); + } + return cont+args[i]+cont; + } ); + } + sys.p(treated); + query_queue.push({sql: treated, callback: callback}); + if (readyState) { + events.emit('ReadyForQuery'); + } + } + }; + + this.prepare = function (query) { + + var r = new process.Promise(); + var name = md5(md5(query)); + var offset = Math.floor(Math.random() * 10); + name = name.replace(/\d/g, "").slice(offset,4+offset); + + var treated = query; + var i = 0; + if (query.match(/\?/)) { + + treated = treated.replace(/\?/g, function (str, p1, offset, s) { + i = i + 1; + return "$"+i; + }); + } + + stmt = "PREPARE " + name + " AS " + treated; + + var conn = this; + + query_queue.push({sql: stmt, callback: function (c) { + // R is going to be null or otherwise weird, as we're emitting + // a PREPARE, instead of anything that would return useful data + // from the database. + var q = new Stmt(name, i, conn ); + r.emitSuccess(q); + } + }); + return r; + }; + this.close = function () { closeState = true; }; + }; - +function Stmt (name, len, conn) { + var stmt = "EXECUTE "+name+" ( "; + var que = []; + for (var i = 1; i<=len; i++) { + que.push("?"); + } + stmt = stmt + que.join(",") + " )"; + + sys.puts(stmt); + + this.execute = function (args, callback) { + if (args.length > len) { + throw new Error("Cannot execute: Too many arguments"); + } + else if (args.length < len) { + // Pad out the length with nulls. + for (var i = args.length; i<= len; i++) { + args.push(null); + } + } + else { + // Nothing to see here. + ; + } + return conn.query(stmt, args, callback); + }; +} \ No newline at end of file From d4e4ef38b70e30296a5a94006f6728ea7c43cc7e Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Wed, 9 Dec 2009 14:24:44 -0800 Subject: [PATCH 09/54] continuing the merge --- README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/README.md b/README.md index cd907a3..84ad517 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ This library is a implementation of the PostgreSQL backend/frontend protocol in javascript. It uses the node.js tcp and event libraries. A javascript md5 library is included for servers that require md5 password hashing (this is default). +This library also allows for the handling of prepared queries and parameterized queries. + ## Example use var sys = require("sys"); @@ -14,3 +16,31 @@ It uses the node.js tcp and event libraries. A javascript md5 library is includ }); db.close(); +## Example use of Parameterized Queries + + var sys = require("sys"); + var pg = require("postgres.js"); + + var db = new pg.Connection("database", "username", "password"); + db.query("SELECT * FROM yourtable WHERE id = ?", [1], function (data) { + + sys.p(data); + }); + db.close(); + +## Example use of Prepared Queries + + var sys = require("sys"); + var pg = require("postgres.js"); + + var db = new pg.Connection("database", "username", "password"); + + db.prepare("SELECT * FROM yourtable WHERE id = ?").addCallback( function (query) { + + sys.p(query); + query.execute(["1"], function (d) { + sys.p(d); + }); + /* More queries here. */ + }); + db.close(); \ No newline at end of file From 8b7541db3503d31892b8b556b12b77bca1f7699b Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Wed, 9 Dec 2009 14:31:25 -0800 Subject: [PATCH 10/54] Finish the merge --- README.markdown | 50 ------------------------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 README.markdown diff --git a/README.markdown b/README.markdown deleted file mode 100644 index 21e4a99..0000000 --- a/README.markdown +++ /dev/null @@ -1,50 +0,0 @@ -# PostgreSQL for Javascript - -This library is a implementation of the PostgreSQL backend/frontend protocol in javascript. -It uses the node.js tcp and event libraries. A javascript md5 library is included for servers that require md5 password hashing (this is default). - -This library also allows for the handling of prepared queries and parameterized queries. -As this library does not use libpq (which supports automatic parameterization), I use Postgres' dollar-quoting system to provide argument wrapping, before sending the query to the database. - -## Example use - - var sys = require("sys"); - var Postgres = require('postgres.js'); - - function onLoad() { - var db = new Postgres.Connection("database", "username", "password"); - db.query("SELECT * FROM sometable", function (data) { - sys.p(data); - }); - db.close(); - } - - -## Example use of Parameterized Queries - - var sys = require("sys"); - var pg = require("postgres.js"); - - var db = new pg.Connection("database", "username", "password"); - db.query("SELECT * FROM yourtable WHERE id = ?", [1], function (data) { - - sys.p(data); - }); - db.close(); - -## Example use of Prepared Queries - - var sys = require("sys"); - var pg = require("postgres.js"); - - var db = new pg.Connection("database", "username", "password"); - - db.prepare("SELECT * FROM yourtable WHERE id = ?").addCallback( function (query) { - - sys.p(query); - query.execute(["1"], function (d) { - sys.p(d); - }); - /* More queries here. */ - }); - db.close(); \ No newline at end of file From c7f2d98540575fc63239aef0bba689bf1f4cd74c Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Sat, 12 Dec 2009 23:20:42 -0800 Subject: [PATCH 11/54] Significant rearchitecture of postgres.js. Moving to a Promise-based external API. Addition of a PostgreSQL protocol Prepared Query metaphor. Prepared Queries now (mostly) work. Starting to move towards a correct disconnect from the server. --- postgres.js | 553 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 431 insertions(+), 122 deletions(-) diff --git a/postgres.js b/postgres.js index 02cb428..2a76e91 100644 --- a/postgres.js +++ b/postgres.js @@ -1,5 +1,22 @@ /*jslint bitwise: true, eqeqeq: true, immed: true, newcap: true, nomen: true, onevar: true, plusplus: true, regexp: true, undef: true, white: true, indent: 2 */ /*globals include md5 node exports */ + +/* Expected flow is: + Query + * RowDescription + * DataRow + * CommandComplete + * ReadyForQuery + Parse + * Parse + * ParseCompleted + Bind + * Bind + * BindCompleted + Execute + * Execute +*/ + process.mixin(require('./lib/md5')); var bits = require('./lib/bits'); @@ -8,12 +25,39 @@ var parsers = require("./lib/parsers"); var tcp = require("tcp"); var sys = require("sys"); -exports.DEBUG = 0; +exports.DEBUG = 1; var postgres_parameters = {}; // http://www.postgresql.org/docs/8.3/static/protocol-message-formats.html var formatter = { + + Bind: function (name, statement, args) { + var b = (new bits.Encoder('B')) + .push_cstring(name) // The name of the bound portal + .push_cstring(statement) // The name of the prepared statement + .push_int16(args.length); // Add the number of format codes. + //.push_int16(0) // Marks them all as text + for (var i = 0; i < args.length; i++) { + b.push_int16(0); // Mark them all text. + } + b.push_int16(args.length); // Add the number of parameters. + + for (var i = 0; i < args.length; i++) { + // Add the size of the parameter. + b.push_int32( process._byteLength(args[i]) ); // Add the length of the argument + b.push_raw_string( args[i] ); // Add the argument itself. + + }; + b.push_int16(1); + b.push_int16(0); + //b.push_int16(0); // Mark that we don't know what the response is going to be. + //b.push_int16(0); + //b.push_int16(0); // All of them should be text. + + return b; + }, + CopyData: function () { // TODO: implement }, @@ -36,14 +80,27 @@ var formatter = { FunctionCall: function () { // TODO: implement }, - Parse: function (name, query, var_types) { + Parse: function (name, query, args) { + if (exports.DEBUG > 0) { + sys.debug("Name is " + name); + sys.debug("Query is " + query); + sys.debug("Args are " + args); + } var builder = (new bits.Encoder('P')) .push_cstring(name) - .push_cstring(query) - .push_int16(var_types.length); - var_types.each(function (var_type) { - builder.push_int32(var_type); - }); + .push_cstring(query); + // sys.p(args); + if (args.length > 0) { + builder.push_int16(args.length); + for (var i = 0; i <= args.length; i++) { + builder.push_int32(0); + } + } + else { + builder.push_int16(0); + // builder.push_int32(0); + // No types. + } return builder; }, PasswordMessage: function (password) { @@ -120,6 +177,15 @@ function parse_response(code, stream) { type = "ParameterStatus"; args = [stream.shift_cstring(), stream.shift_cstring()]; break; + case 't': + type = "ParameterDescription"; + var len = stream.shift_int16(); + var data = []; + for (var i = 0; i < len; i++) { + data.push( stream.shift_int32 ); + } + args = data; + break; case 'K': type = "BackendKeyData"; args = [stream.shift_int32(), stream.shift_int32()]; @@ -163,6 +229,17 @@ function parse_response(code, stream) { type = "CommandComplete"; args = [stream.shift_cstring()]; break; + + case '1': // Parse Complete. + if (exports.DEBUG > 0) { + sys.debug("Got a ParseComplete message."); + } + type = "ParseComplete"; + args = [stream.shift_int32]; + break; + case '2': + type = "BindComplete"; + break; } if (!type) { sys.debug("Unknown response " + code); @@ -171,7 +248,7 @@ function parse_response(code, stream) { } -exports.Connection = function (database, username, password, port, host) { +exports.connect = function (database, username, password, port, host) { var connection, events, query_queue, row_description, query_callback, results, readyState, closeState; // Default to port 5432 @@ -189,17 +266,29 @@ exports.Connection = function (database, username, password, port, host) { query_queue = []; readyState = false; closeState = false; - + needsFlush = false; + msg_queue = []; + + var cQuery; // This is the currently executing query. + + canQuery = false; + // Sends a message to the postgres server function sendMessage(type, args) { - var stream = (formatter[type].apply(this, args)).toString(); - if (exports.DEBUG > 0) { - sys.debug("Sending " + type + ": " + JSON.stringify(args)); - if (exports.DEBUG > 2) { - sys.debug("->" + JSON.stringify(stream)); + + if (exports.DEBUG > 0) { + sys.debug("Type is " + type); + sys.debug("Args are " + args); } - } - connection.send(stream, "binary"); + + var stream = (formatter[type].apply(this, args)).toString(); + if (exports.DEBUG > 0) { + sys.debug("Sending " + type + ": " + JSON.stringify(args)); + if (exports.DEBUG >= 2) { + sys.debug("->" + JSON.stringify(stream)); + } + } + connection.send(stream, "binary"); } // Set up tcp client @@ -207,8 +296,13 @@ exports.Connection = function (database, username, password, port, host) { connection.addListener("connect", function () { sendMessage('StartupMessage', [{user: username, database: database}]); }); + + // connection.addListener("receive", function (data) { var input, code, len, stream, command; + if (exports.DEBUG > 1){ + sys.debug("Got a response. Attempting to decode."); + } input = new bits.Decoder(data); if (exports.DEBUG > 2) { sys.debug("<-" + JSON.stringify(data)); @@ -223,64 +317,181 @@ exports.Connection = function (database, username, password, port, host) { } command = parse_response(code, stream); if (command.type) { - if (exports.DEBUG > 0) { + if (exports.DEBUG > 1) { sys.debug("Received " + command.type + ": " + JSON.stringify(command.args)); } command.args.unshift(command.type); - events.emit.apply(events, command.args); + if (cQuery != null && cQuery.events != undefined) { + if (exports.DEBUG > 0) { + sys.debug ("Current query is not null and current query is " + cQuery.type); + } + if (cQuery.events.listeners(command.type).length >= 1) { + cQuery.events.emit.apply(cQuery.events, command.args); + } + else { + events.emit.apply(events, command.args); + } + } + else { + events.emit.apply(events, command.args); + } } } }); + // connection.addListener("eof", function (data) { connection.close(); }); + // connection.addListener("disconnect", function (had_error) { if (had_error) { sys.debug("CONNECTION DIED WITH ERROR"); } }); - + // Set up callbacks to automatically do the login events.addListener('AuthenticationMD5Password', function (salt) { var result = "md5" + md5(md5(password + username) + salt); sendMessage('PasswordMessage', [result]); }); + // events.addListener('AuthenticationCleartextPassword', function () { sendMessage('PasswordMessage', [password]); }); + // events.addListener('ErrorResponse', function (e) { if (e.S === 'FATAL') { sys.debug(e.S + ": " + e.M); - connection.close(); + connection.close(); // Well, that's bad. + } + if (e.S === "ERROR") { + // var err = new Error(); + // err.name = "Error"; + // err.message = e.M; + // throw err; + sys.p(e); + msg = new String(e.M); + throw new Error("DB Error: " + msg); } }); + // + events.addListener('ParameterStatus', function(key, value) { - postgres_parameters[key] = value; + postgres_parameters[key] = value; }); + + function queue (msg) { + msg_queue.push(msg); + } + + function flush (promise, eventHandler) { + // We now flush all entries in the queue. + // We iterate the query_queue, send each message, and finally use the + // object provided us as the Event responder for this query set. + + var q = {type:"Flush", events:eventHandler, promise: promise}; + msg_queue.push(q); + sys.debug(canQuery); + + if (canQuery) { + events.emit("FlushBuffer"); + } + } + + function send (msg) { + + if (canQuery && !closeState) { + // immediately send the message + canQuery = false; + cQuery = msg; + sendMessage(msg.type, cQuery.args); + } + else if (closeState) { + // Raise an error + throw new Error("Cannot execute queries on closed handle."); + } + else { + // Push it onto the stack, and return. + msg_queue.push(msg); + } + } + + // The main query buffer; this handles *all* query traffic. + + events.addListener("FlushBuffer", function () { + + if (msg_queue.length > 0) { + canQuery = false; + var i; + var len = msg_queue.length; + for (i = 0; i < len; i++) { + var Query = msg_queue.shift(); + if (exports.DEBUG > 0) { + sys.debug("RFQ: "+Query.type); + } + if (Query.type == "Flush") { + cQuery = Query; + sendMessage(Query.type, Query.args); + break; // Exit this loop. + } + else { + // Just send the message. + sendMessage(Query.type, Query.args); + } + } + // var Query = msg_queue.shift(); // Grab the first. + + // query_callback = query.callback || function () {}; + } else { + // if (closeState) { + // // This is not how we should be shutting down. + // + // connection.close(); + // } else { + canQuery = true; + cQuery = null; // Done and Handled. + } + + }); + + events.addListener('ReadyForQuery', function () { - if (query_queue.length > 0) { - var query = query_queue.shift(); - query_callback = query.callback || function() {}; - sendMessage('Query', [query.sql]); - readyState = false; - } else { - if (closeState) { - connection.close(); + + if (msg_queue.length > 0) { + events.emit("FlushBuffer"); } else { - readyState = true; + if (closeState) { + // This is not how we should be shutting down. + sys.debug("Got shutdown."); + msg_queue.push({type:"Terminate", args:[]}); + connection.close(); // Shut it all down. + } } - } + // // Do we have anything on the buffer? + // if (msg_queue.length > 0) { + // events.emit("FlushBuffer"); + // } + // else if (closeState) { + // msg_queue.push({type:"Terminate", }); + // } + }); + events.addListener("RowDescription", function (data) { - row_description = data; - results = []; + cQuery.row_description = data; + cQuery.results = []; }); + events.addListener("DataRow", function (data) { + dataParser(data, cQuery); // use the global query context. + }); + + function dataParser (data, mQuery) { var row, i, l, description, value; row = {}; l = data.length; for (i = 0; i < l; i += 1) { - description = row_description[i]; + description = mQuery.row_description[i]; value = data[i]; if (value !== null) { // TODO: investigate to see if these numbers are stable across databases or @@ -308,117 +519,215 @@ exports.Connection = function (database, username, password, port, host) { } row[description.field] = value; } - results.push(row); - }); + mQuery.results.push(row); + }; events.addListener('CommandComplete', function (data) { - query_callback.call(this, results); + // cQuery.promise.emitSuccess(data); + cQuery.promise.emitSuccess( cQuery.results ); + //query_callback.call(this, results); }); - - this.query = function (sql, args, callback) { - - if (callback == null) { - - // This has no parameters to manipulate. - - // Assume the args value is the callback - sys.puts("Null callback"); - sys.p(args); - query_queue.push({sql: sql, callback: args || function () {} }); - if (readyState) { - events.emit('ReadyForQuery'); - } + + events.addListener("Notice", function (data) { + // This is a string message from PG - not an error. + sys.puts(d.S + " :: " + d.M); + }); + + + this.query = function (sql, args) { + var p = new process.Promise(); + if (sql.match(/\?/)) { + // We will do an anonymous prepared statement. + ; } else { - // We have an args list. - // This means, we have to map our ?'s and test for a variety of - // edge cases. - sys.puts("Got args."); - var i = 0; - var slice = md5(md5(sql)); - //sys.p(slice); - var offset = Math.floor(Math.random() * 10); - cont = "$" + slice.replace(/\d/g, "").slice(offset,4+offset) + "$"; - var treated = sql; - sys.p(cont); - if (sql.match(/\?/)) { - treated = sql.replace(/\?/g, function (str, offset, s) { - if (!args[i]) { - // raise an error - throw new Error("Argument "+i+" does not exist!"); - } - return cont+args[i]+cont; - } ); - } - sys.p(treated); - query_queue.push({sql: treated, callback: callback}); - if (readyState) { - events.emit('ReadyForQuery'); - } + // We can just emit a simple query. + + queue({type:"Query", args:[sql], promise: p}); + return p; } - }; this.prepare = function (query) { - var r = new process.Promise(); - var name = md5(md5(query)); - var offset = Math.floor(Math.random() * 10); - name = name.replace(/\d/g, "").slice(offset,4+offset); + var p = new process.Promise(); var treated = query; var i = 0; + var arglist = new Array(); + // Replace all ?'s with bind parameters. if (query.match(/\?/)) { - treated = treated.replace(/\?/g, function (str, p1, offset, s) { i = i + 1; + arglist.push(0); // a null string, representing that it is, in fact, an item. return "$"+i; }); } + var name = md5(md5(query)); + var offset = Math.floor(Math.random() * 10); + name = "postgres_js_prepared_" + name.replace(/\d/g, "").slice(offset,4+offset); - stmt = "PREPARE " + name + " AS " + treated; - - var conn = this; + // Assumes, for the moment, that all the bind variables are going to be + // text, as we otherwise don't know what they are. - query_queue.push({sql: stmt, callback: function (c) { - // R is going to be null or otherwise weird, as we're emitting - // a PREPARE, instead of anything that would return useful data - // from the database. - var q = new Stmt(name, i, conn ); - r.emitSuccess(q); - } + var e = new process.EventEmitter(); + var Stmt = new Statement(name, i, p); + e.addListener("ParseComplete", function (data) { + // Our next message is (probably) going to be a ReadyForQuery. + canQuery = true; // Say that we now allow for more queries on the wire. + if (exports.DEBUG > 0) { + sys.debug("Prepare:: ParseComplete message received."); + } + p.emitSuccess(); }); - return r; + + queue( {type:"Parse", args:[name, treated, []] } ); + + var rowDesc = []; + flush(p, e); // Issues the implied Flush command. + + //return p; + return Stmt; }; + function Statement (name, len, prepared) { + // Execute sets up a bind, and then executes the statement. + //queue( {type:"Describe", args:[name] }); + + // Until it's prepared, we won't ever Flush. This all becomes + // asynchronous + + var format; + var row_description = []; + + var isPrepared = false; + var iBuffer = []; + + var isDescribed = false; + + prepared.addCallback(function () { + if (exports.DEBUG > 0) { + sys.debug("Statement:: Prepare completed."); + } + isPrepared = true; + // If there's any buffered executes, perform them now. + // + if (iBuffer.length > 0) { + if (exports.DEBUG > 0) { + sys.debug("Found internal buffer."); + } + for (var i = 0; i < iBuffer.length; i++) { + var o = iBuffer[i]; + if (o.type == "Flush") { + if (exports.DEBUG > 0) { + sys.debug("Flushing."); + } + flush(o.promise, o.events); + } + else { + if (exports.DEBUG > 0) { + sys.debug("Queuing "+ o.type); + } + queue(o); + } + } + iBuffer = []; // Zero it. + } + }); + + this.execute = function (args) { + + if (args.length > len) { + throw new Error("Cannot execute: Too many arguments"); + } + else if (args.length < len) { + + // We need to pad out the length with nulls + for (var i = args.length; i<= len; i++) { + args.push(null); + } + } + var e = new process.EventEmitter(); + var promise = new process.Promise(); + + // var our = this; + // our.results = []; + var our = new Object(); + var obj = this; + our.results = []; + our.row_description = null; + e.addListener("BindComplete", function (data) { + // This is good, but we don't need to do anything as a result. + if (exports.DEBUG > 0) { + sys.debug("got Bind Complete"); + } + }); + + e.addListener("ParameterDescription", function (data) { + // Defines what our system actually needs to send. For the + // moment, we don't need to worry about this. + if (exports.DEBUG > 0) { + sys.debug("got Parameter Description"); + } + }); + + e.addListener("RowDescription", function (data) { + if (exports.DEBUG > 0) { + sys.debug("got Row Description"); + } + obj.row_description = data; + our.results = []; + isDescribed = true; + }); + + e.addListener("DataRow", function (data) { + if (exports.DEBUG > 0) { + sys.debug("got a Data Row"); + } + if (our.row_description == null) { + our.row_description = obj.row_description; + } + dataParser(data, our); + }); + + e.addListener("CommandComplete", function (data) { + if (exports.DEBUG > 0) { + sys.debug("got CommandComplete"); + } + promise.emitSuccess(our.results); + events.emit("ReadyForQuery"); + }); + + + if (isPrepared == true) { + + if (!(isDescribed)) { + queue( {type:"Describe", args:[name, "S"] } ); + isDescribed = true; + } + // We use the main buffer for queries. + queue( {type:"Bind", args:["", name, args] } ); + queue( {type:"Execute", args:["", 0] } ); + flush(promise, e); // Issues the implied Flush command. + } + else if (isPrepared == false) { + // we buffer internally until we get the main buffering response. + + if (!(isDescribed)) { + iBuffer.push( {type:"Describe", args:[name, "S"] } ); + isDescribed = true; + } + + iBuffer.push( {type:"Bind", args:["", name, args] } ); + iBuffer.push( {type:"Execute", args:["", 0] } ); + iBuffer.push( {type:"Flush", events: e, promise: p } ); + } + return promise; // Return our Promise. + }; + } + this.close = function () { - closeState = true; + // This needs to be updated to handle a DB-side close + closeState = true; }; -}; - -function Stmt (name, len, conn) { - var stmt = "EXECUTE "+name+" ( "; - var que = []; - for (var i = 1; i<=len; i++) { - que.push("?"); - } - stmt = stmt + que.join(",") + " )"; - - sys.puts(stmt); - - this.execute = function (args, callback) { - if (args.length > len) { - throw new Error("Cannot execute: Too many arguments"); - } - else if (args.length < len) { - // Pad out the length with nulls. - for (var i = args.length; i<= len; i++) { - args.push(null); - } - } - else { - // Nothing to see here. - ; - } - return conn.query(stmt, args, callback); - }; +}; \ No newline at end of file From d129174201b1c920247dc45f0048352c80aaa8bd Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Tue, 15 Dec 2009 21:14:41 -0800 Subject: [PATCH 12/54] Almost with a working implementation of the protocol. --- lib/bits.js | 5 +- lib/parsers.js | 28 ++++-- postgres.js | 250 +++++++++++++++++++++++++++++++------------------ 3 files changed, 181 insertions(+), 102 deletions(-) diff --git a/lib/bits.js b/lib/bits.js index 4914bdd..b06abfd 100644 --- a/lib/bits.js +++ b/lib/bits.js @@ -33,7 +33,10 @@ // Add a postgres header to the binary string and return it. proto.toString = function () { - return this.header + encode_int32(this.data.length + 4) + this.data; + return this.header + // encode_int32(this.data.length + 4) + + + encode_int32(process._byteLength(this.data) + 4) + + this.data; }; // Encode number as 32 bit 2s compliment diff --git a/lib/parsers.js b/lib/parsers.js index e1833e3..e5acadf 100644 --- a/lib/parsers.js +++ b/lib/parsers.js @@ -1,6 +1,6 @@ process.mixin(GLOBAL, require("sys")); var oids = require('./type-oids'); - +exports.DEBUG = 0; // DATE parsing // parse a string like YYYY-MM-DD into @@ -18,12 +18,16 @@ var tz_regex = { } exports.parseDateFromPostgres = function (str,datestyle,OID) { - debug("parsing string: "+ str + " with datestyle: " + datestyle + " with OID: " + OID); + if (exports.DEBUG > 0) { + debug("parsing string: "+ str + " with datestyle: " + datestyle + " with OID: " + OID); + } + var style = datestyle.split(','); var order = style[1]; style = style[0].replace(/^\s+|\s+$/,''); - if (!(style in date_regex)) { + if (!(style in date_regex) && (exports.DEBUG > 0)) { + sys.debug("Error datestyle not implemented: " + style); } @@ -31,23 +35,31 @@ exports.parseDateFromPostgres = function (str,datestyle,OID) { switch(OID) { case oids.TIMESTAMPTZ: tz = str.replace(tz_regex[style],tz_regex[style+'r']); - debug("Timezone: " + tz); + if (exports.DEBUG > 0) { + debug("Timezone: " + tz); + } if (tz == str) tz = ''; case oids.TIMESTAMP: case oids.DATE: date = str.replace(date_regex[style],date_regex[style+'r']); - debug("date: " + date); + if (exports.DEBUG > 0) { + debug("date: " + date); + } if (date == str) date = ''; if (OID==oids.DATE) break; case oids.TIME: time = ' ' + str.replace(time_regex[style],time_regex[style+'r']); - debug("time: " + time); + if (exports.DEBUG > 0) { + debug("time: " + time); + } if (time == str) time = ''; } date = ((date=='')?'January 1, 1970':date) + ((time=='')?'':' ') + time + ((tz=='')?'':' ') + tz; - - debug("created date: " + date); + + if (exports.DEBUG > 0) { + debug("created date: " + date); + } var d = new Date(); d.setTime(Date.parse(date)); diff --git a/postgres.js b/postgres.js index 2a76e91..20b3565 100644 --- a/postgres.js +++ b/postgres.js @@ -25,7 +25,7 @@ var parsers = require("./lib/parsers"); var tcp = require("tcp"); var sys = require("sys"); -exports.DEBUG = 1; +exports.DEBUG = 0; var postgres_parameters = {}; @@ -263,15 +263,16 @@ exports.connect = function (database, username, password, port, host) { connection = tcp.createConnection(port, host=t_host); events = new process.EventEmitter(); - query_queue = []; + readyState = false; - closeState = false; - needsFlush = false; - msg_queue = []; var cQuery; // This is the currently executing query. - canQuery = false; + // msg_queue = []; // Our query buffer. + msg_queue = new Buffer(); + canQuery = false; // Whether or not we can push new queries onto the wire. + closeState = false; // Is our connection to be closed? If so, we + // shouldn't accept new messages. // Sends a message to the postgres server function sendMessage(type, args) { @@ -361,7 +362,9 @@ exports.connect = function (database, username, password, port, host) { // events.addListener('ErrorResponse', function (e) { if (e.S === 'FATAL') { + sys.debug(e.S + ": " + e.M); + connection.close(); // Well, that's bad. } if (e.S === "ERROR") { @@ -384,14 +387,7 @@ exports.connect = function (database, username, password, port, host) { msg_queue.push(msg); } - function flush (promise, eventHandler) { - // We now flush all entries in the queue. - // We iterate the query_queue, send each message, and finally use the - // object provided us as the Event responder for this query set. - - var q = {type:"Flush", events:eventHandler, promise: promise}; - msg_queue.push(q); - sys.debug(canQuery); + function flush () { if (canQuery) { events.emit("FlushBuffer"); @@ -418,36 +414,27 @@ exports.connect = function (database, username, password, port, host) { // The main query buffer; this handles *all* query traffic. + msg_queue.each(function (Q) { + + if (Q.msgs.length > 0) { + cQuery = Q; + for (var i = 0; i 0) { - canQuery = false; - var i; - var len = msg_queue.length; - for (i = 0; i < len; i++) { - var Query = msg_queue.shift(); - if (exports.DEBUG > 0) { - sys.debug("RFQ: "+Query.type); - } - if (Query.type == "Flush") { - cQuery = Query; - sendMessage(Query.type, Query.args); - break; // Exit this loop. - } - else { - // Just send the message. - sendMessage(Query.type, Query.args); - } + + if (canQuery == true) { + canQuery = false; + msg_queue.next(); } - // var Query = msg_queue.shift(); // Grab the first. - - // query_callback = query.callback || function () {}; + } else { - // if (closeState) { - // // This is not how we should be shutting down. - // - // connection.close(); - // } else { canQuery = true; cQuery = null; // Done and Handled. } @@ -457,24 +444,18 @@ exports.connect = function (database, username, password, port, host) { events.addListener('ReadyForQuery', function () { + canQuery = true; if (msg_queue.length > 0) { events.emit("FlushBuffer"); } else { if (closeState) { // This is not how we should be shutting down. - sys.debug("Got shutdown."); - msg_queue.push({type:"Terminate", args:[]}); - connection.close(); // Shut it all down. + // sys.debug("Got shutdown."); + // msg_queue.push({type:"Terminate", args:[]}); + // events.emit("FlushBuffer"); + // connection.close(); // Shut it all down. } } - // // Do we have anything on the buffer? - // if (msg_queue.length > 0) { - // events.emit("FlushBuffer"); - // } - // else if (closeState) { - // msg_queue.push({type:"Terminate", }); - // } - }); events.addListener("RowDescription", function (data) { @@ -542,7 +523,12 @@ exports.connect = function (database, username, password, port, host) { else { // We can just emit a simple query. - queue({type:"Query", args:[sql], promise: p}); + queue({ + promise: p, + msgs: [ + { type:"Query", args:[sql] }, // Add the one message to the internal buffer. + ] + }); return p; } }; @@ -580,10 +566,17 @@ exports.connect = function (database, username, password, port, host) { p.emitSuccess(); }); - queue( {type:"Parse", args:[name, treated, []] } ); + queue({ + promise: p, + events: e, + msgs: [ + { type:"Parse", args:[name, treated, []] }, // Adds a single message to the internal buffer. + { type:"Flush", args:[] }, + ] + }); var rowDesc = []; - flush(p, e); // Issues the implied Flush command. + flush(); // Issues the implied Flush command. //return p; return Stmt; @@ -608,30 +601,22 @@ exports.connect = function (database, username, password, port, host) { if (exports.DEBUG > 0) { sys.debug("Statement:: Prepare completed."); } + canQuery = true; isPrepared = true; + flush(); // If there's any buffered executes, perform them now. // - if (iBuffer.length > 0) { - if (exports.DEBUG > 0) { - sys.debug("Found internal buffer."); - } - for (var i = 0; i < iBuffer.length; i++) { - var o = iBuffer[i]; - if (o.type == "Flush") { - if (exports.DEBUG > 0) { - sys.debug("Flushing."); - } - flush(o.promise, o.events); - } - else { - if (exports.DEBUG > 0) { - sys.debug("Queuing "+ o.type); - } - queue(o); - } - } - iBuffer = []; // Zero it. - } + // if (iBuffer.length > 0) { + // if (exports.DEBUG > 0) { + // sys.debug("Found internal buffer."); + // } + // for (var i = 0; i < iBuffer.length; i++) { + // var o = iBuffer[i]; + // queue(o); + // } + // flush(); + // iBuffer = []; // Zero it. + // } }); this.execute = function (args) { @@ -698,28 +683,31 @@ exports.connect = function (database, username, password, port, host) { }); + var msgs = []; + + if (!(isDescribed)) { + msgs.push( {type:"Describe", args:[name, "S"] } ); + isDescribed = true; + } + // We use the main buffer for queries. + msgs.push( {type:"Bind", args:["", name, args] } ); + msgs.push( {type:"Execute", args:["", 0] } ); + msgs.push( {type:"Flush", args:[] } ); + + queue({ + events: e, + promise: p, + msgs:msgs + }); + if (isPrepared == true) { - if (!(isDescribed)) { - queue( {type:"Describe", args:[name, "S"] } ); - isDescribed = true; - } - // We use the main buffer for queries. - queue( {type:"Bind", args:["", name, args] } ); - queue( {type:"Execute", args:["", 0] } ); - flush(promise, e); // Issues the implied Flush command. + flush(); // Issues the implied Flush command. } else if (isPrepared == false) { // we buffer internally until we get the main buffering response. - - if (!(isDescribed)) { - iBuffer.push( {type:"Describe", args:[name, "S"] } ); - isDescribed = true; - } - - iBuffer.push( {type:"Bind", args:["", name, args] } ); - iBuffer.push( {type:"Execute", args:["", 0] } ); - iBuffer.push( {type:"Flush", events: e, promise: p } ); + canQuery = false; // No more queries until we're ready. + } return promise; // Return our Promise. }; @@ -727,7 +715,83 @@ exports.connect = function (database, username, password, port, host) { this.close = function () { // This needs to be updated to handle a DB-side close - closeState = true; + + queue({ msgs:[{type:"Terminate", args:[]}] }); + connection.close(); // G'bye. + + }; + this.commit = function () { + var p = new process.Promise(); + queue({ msgs:[{type:"Sync", args:[]}], promise:p }); + return p; }; -}; \ No newline at end of file + this.rollback = function () { + return this.query("ROLLBACK"); + }; + +}; + + +function queue () { + + if (mainBuffer.hasElements) { + mainBuffer.each(function (i) { + // Registers the sendNextItem + // Now, if callbacks end up pushing additional statements onto + // their buffers, I will end up with another message hitting + // this buffer. + sendMessage(i); + }); + } +} + + +function Buffer () { + var buffer = []; + var position = 0; + var hasEnded = false; + this.e = new process.EventEmitter(); + this.push = function (msg) { + buffer.push(msg); + this.length++; + if (hasEnded) { + hasEnded = false; + this.e.emit("nextItem"); + } + this.e.emit("itemPushed"); + + } + this.length = 0; + + this.next = function () { + this.e.emit("nextItem"); + } + + this.unshift = function () { + pointer--; + } + + this.e.addListener("nextItem", function () { + if (buffer[position] !== undefined) { + // sys.puts(position); + // position = position + 1; + this.emit("sendNextItem", buffer[position++]); + + } + else { + hasEnded = true; + this.emit("eof"); + } + }); + // Expects to have the caller register itself as the nextItem callback. + + this.each = function(callback) { + this.e.addListener("sendNextItem", function (i) { + // sys.puts(position); + callback(i); + // this.emit("nextItem"); + }); + // this.e.emit("nextItem"); + } +} \ No newline at end of file From 3c2721b5c569b26f4256362687355e10667ac7e3 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Tue, 15 Dec 2009 21:43:03 -0800 Subject: [PATCH 13/54] Initial prepared statements via queries are working. Code is pretty basic. :) --- postgres.js | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/postgres.js b/postgres.js index 20b3565..24bb58d 100644 --- a/postgres.js +++ b/postgres.js @@ -290,6 +290,10 @@ exports.connect = function (database, username, password, port, host) { } } connection.send(stream, "binary"); + if (cQuery && cQuery.events) { + cQuery.events.emit("messageSent"); + } + } // Set up tcp client @@ -604,19 +608,6 @@ exports.connect = function (database, username, password, port, host) { canQuery = true; isPrepared = true; flush(); - // If there's any buffered executes, perform them now. - // - // if (iBuffer.length > 0) { - // if (exports.DEBUG > 0) { - // sys.debug("Found internal buffer."); - // } - // for (var i = 0; i < iBuffer.length; i++) { - // var o = iBuffer[i]; - // queue(o); - // } - // flush(); - // iBuffer = []; // Zero it. - // } }); this.execute = function (args) { @@ -716,13 +707,17 @@ exports.connect = function (database, username, password, port, host) { this.close = function () { // This needs to be updated to handle a DB-side close - queue({ msgs:[{type:"Terminate", args:[]}] }); - connection.close(); // G'bye. + var e = new process.EventEmitter(); + e.addListener("messageSent", function () { + connection.close(); // G'bye. + }); + queue({ msgs:[{type:"Terminate", args:[]}], events: e }); + }; this.commit = function () { var p = new process.Promise(); - queue({ msgs:[{type:"Sync", args:[]}], promise:p }); + queue({ msgs:[{type:"Sync", args:[]}], promise: p }); return p; }; From f6130ea9977f3e3528d450ac045c726567c543ea Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Tue, 15 Dec 2009 23:02:19 -0800 Subject: [PATCH 14/54] Updating the documentation to reflect current status. --- README.md | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 84ad517..6f914ec 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,16 @@ This library is a implementation of the PostgreSQL backend/frontend protocol in javascript. It uses the node.js tcp and event libraries. A javascript md5 library is included for servers that require md5 password hashing (this is default). -This library also allows for the handling of prepared queries and parameterized queries. +This library allows for the handling of prepared queries. + +Parameterized queries are (currently) no longer supported. + +If you wish to nest DB calls, db.close must be in the deepest callback, or all statements that occur inside of callbacks deeper than the callback which handles db.close will not be executed. ## Example use var sys = require("sys"); - var Postgres = require("./postgres"); + var Postgres = require("postgres"); var db = new Postgres.Connection("database", "username", "password"); db.query("SELECT * FROM sometable", function (data) { @@ -19,7 +23,7 @@ This library also allows for the handling of prepared queries and parameterized ## Example use of Parameterized Queries var sys = require("sys"); - var pg = require("postgres.js"); + var pg = require("postgres"); var db = new pg.Connection("database", "username", "password"); db.query("SELECT * FROM yourtable WHERE id = ?", [1], function (data) { @@ -31,16 +35,16 @@ This library also allows for the handling of prepared queries and parameterized ## Example use of Prepared Queries var sys = require("sys"); - var pg = require("postgres.js"); + var pg = require("postgres"); var db = new pg.Connection("database", "username", "password"); - db.prepare("SELECT * FROM yourtable WHERE id = ?").addCallback( function (query) { - - sys.p(query); - query.execute(["1"], function (d) { - sys.p(d); - }); - /* More queries here. */ + var stmt = db.prepare("SELECT * FROM yourtable WHERE id = ?"); + + stmt.execute([1]).addCallback(function (d) { + + sys.p(d); + db.close(); }); - db.close(); \ No newline at end of file + + From 26e16a4eb404b0a2ca43a2d40e8c94432c051a01 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Mon, 2 Aug 2010 09:02:09 -0700 Subject: [PATCH 15/54] Removal of the sql.js requirement; will remove the file shortly. Significant upgrades to the connection logic; now (mostly) implements prepared queries and parameterized queries by default. .query has been refactored to use the prepared query logic (in the event of parameters); otherwise, it uses the default Query message. Messages are now handled as multi-part blocks pushed onto the wire. This is done to allow for a complete multi-part message (IE, prepared statements) to be messaged without the driver attempting to perform messages out of sequence. --- lib/postgres-pure.js | 734 ++++++++++++++++++++++++++++++------------- 1 file changed, 514 insertions(+), 220 deletions(-) diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 678f288..ae9a8dc 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -24,7 +24,7 @@ THE SOFTWARE. var md5 = require('./md5'); var net = require("net"); var sys = require("sys"); -var sqllib = require('./sql'); +//var sqllib = require('./sql'); var url = require('url'); var Buffer = require('./buffer_extras'); @@ -173,15 +173,15 @@ function parse_response(code, buffer) { num_fields = reader.int16(); data = []; for (i = 0; i < num_fields; i += 1) { - data.push({ - field: reader.cstring(), - table_id: reader.int32(), - column_id: reader.int16(), - type_id: reader.int32(), - type_size: reader.int16(), - type_modifier: reader.int32(), - format_code: reader.int16() - }); + data.push({ + field: reader.cstring(), + table_id: reader.int32(), + column_id: reader.int16(), + type_id: reader.int32(), + type_size: reader.int16(), + type_modifier: reader.int32(), + format_code: reader.int16() + }); } args = [data]; break; @@ -218,238 +218,532 @@ function parse_response(code, buffer) { } -function Connection(args) { - if (typeof args === 'string') { - args = url.parse(args); - args.database = args.pathname.substr(1); - args.auth = args.auth.split(":"); - args.username = args.auth[0]; - args.password = args.auth[1]; - } - var started, conn, connection, events, query_queue, row_description, query_callback, results, readyState, closeState; - - // Default to port 5432 - args.port = args.port || 5432; - - // Default to host 127.0.0.1 - args.hostname = args.hostname || "127.0.0.1"; - +function Query(sql) { + this.sql = sql; + var q = this; + var row_description, results; + results = []; + /* + Returns the next query object in this object buffer. + This can be null. + */ + var arr = [ + { + type: 'Query', + args: [sql] + }, + { + type: 'Flush', + args: [] + } + ]; + var arrpos = 0; + q.next = function () { + arrpos += 1; + return arr[arrpos-1]; + }; + + q.addListener("RowDescription", function (desc) { + row_description = desc; + }); + q.addListener("newRow", function (row)) +} - connection = net.createConnection(args.port, args.hostname); - events = new process.EventEmitter(); - query_queue = []; - readyState = false; - closeState = false; - started = false; - conn = this; - - // Disable the idle timeout on the connection - connection.setTimeout(0); +Query.prototype = new process.EventEmitter; - // Sends a message to the postgres server - function sendMessage(type, args) { - var buffer = (formatter[type].apply(this, args)).frame(); - if (exports.DEBUG > 0) { - sys.debug("Sending " + type + ": " + JSON.stringify(args)); - if (exports.DEBUG > 2) { - sys.debug("->" + buffer.inspect().replace('<', '[')); - } - } - connection.write(buffer); - } - - var queue = []; - function checkInput() { - if (queue.length === 0) { return; } - var first = queue[0]; - var code = String.fromCharCode(first[0]); - var length = first.int32Read(1) - 4; - - // Make sure we have a whole message, TCP comes in chunks - if (first.length < length + 5) { - if (queue.length > 1) { - // Merge the first two buffers - queue.shift(); - var b = new Buffer(first.length + queue[0].length); - first.copy(b); - queue[0].copy(b, first.length); - queue[0] = b; - return checkInput(); - } else { - return; - } +function Prepared(sql, conn, callback) { + + var prepared_name = md5(sql); // Use the md5 hash. This is easily selectable later. + var q = this; + q.row_description = null; + arr.push(); + + var arr = [ + { + type: "Parse", + // Prepared name, the query, and a zero-length array to declare no types. + args: [prepared_name, sql, []], + }, + { + type: "Describe", + args: ['S', prepared_name], + }, + { + type: "Flush", + args: ['S', prepared_name], + }, + ]; // This describes a (nearly) complete lifecycle of a prepared statement. + + + + q.addListener("ParseComplete", function () { + // Execute can now be run successfully. + // Until this point, we can't assume that there's a matching query. + // Anyway, we now run a DESCRIBE operation, and store the row + // description in our object. + // Later optimization might hold on to these objects as hashes in the + // connection object. + conn.next(); + }); + + q.addListener("RowDescription", function (args) { + q.row_description = args; + }); + + /* + Executing the function tests whether or not + we've been passed an argument list. + If we have been, then we need to issue a BIND on the wire. + If we haven't been, we can move straight to EXECUTE. + */ + q.execute = function () { + // If the first argument is an array, then we use that as our bind + // parameters. Otherwise, arguments[0] should be a function. + if (arguments[0] instanceof Array) { + if (arguments[0].length >= 1) { + var callback = null; + // If the functions' a function, yay! + if (typeof(arguments[1]) == 'function') { + callback = arguments[1]; + } + } + } + else if (typeof(arguments[0]) == 'function' ) { + var eP = new execPrepared(portal, prepared_name, [], arguments[0]); + // Announcing that my slot is released in favour of this new query. + // The connection doesn't advance the buffer, but *does* replace + // the current query with this one. + conn.yield_to(eP); + } + else { + q.emit("error", "First argument must be array or function!"); + } } - var message = first.slice(5, 5 + length); - if (first.length === 5 + length) { - queue.shift(); - } else { - queue[0] = first.slice(length + 5, first.length); + + q.next() { + if (arr[pos + 1] !== null) { + pos = pos + 1; + return arr[pos]; + } + } +} +Prepare.prototype = new EventEmitter; - if (exports.DEBUG > 1) { - sys.debug("stream: " + code + " " + message.inspect()); - } - command = parse_response(code, message); - if (command.type) { - if (exports.DEBUG > 0) { - sys.debug("Received " + command.type + ": " + JSON.stringify(command.args)); - } - command.args.unshift(command.type); - events.emit.apply(events, command.args); +function execPrepared = function (portal, prepared, args, callback) { + var q = this; + var results = []; + var arr = [ + { + type: "Execute", + args: [portal, 0], // No limit. Get all the rows. + } + ]; + // If we have args, unshift the Bind. + if (args instanceof Array && args.length >= 1) { + arr.unshift({ + type: "Bind", + args:[portal, prepared, args], + callback: callback + }); } - checkInput(); - } - // Set up tcp client - connection.addListener("connect", function () { - sendMessage('StartupMessage', [{user: args.username, database: args.database}]); - }); - connection.addListener("data", function (data) { - if (exports.DEBUG > 2) { - sys.debug("<-" + data.inspect()); - } - queue.push(data); - checkInput(); - }); - connection.addListener("end", function (data) { - connection.end(); - }); - connection.addListener("disconnect", function (had_error) { - if (had_error) { - sys.debug("CONNECTION DIED WITH ERROR"); + q.addListener("BindComplete", function (args) { + // We can mostly just ignore this, right? It's a notification of + // okay-ness. + conn.next(); + }); + + // Named this instead of DataRow to prevent the main loop from using this, + // instead of the main connection datarow parser. + + q.addListener("newRow", function (row) { + results.push(row); + }); + + q.addListener("CommandComplete", function (data) { + // Whatever we just did is ended. + // If it was a SELECT, args will be an array of rows, + // If it was an INSERT, etc, it'll be the type, and the number of + // affected rows. + if (results.length >= 1) { + callback.call(data, results); + } + else { + this.type = data.type; + callback.call(data); + } + + }); +} + +execPrepared.prototype = new process.EventEmitter; + + +/* Initializes a connection to the database. +DB connections are of the form: + +pgsql://user:password@hostname:port/databasename + +*/ + +function Connection(args) { + if (typeof args === 'string') { + args = url.parse(args); + args.database = args.pathname.substr(1); + args.auth = args.auth.split(":"); + args.username = args.auth[0]; + args.password = args.auth[1]; } - }); + var started, conn, connection, events, query_queue, current_query, results, readyState, closeState; + + // Default to port 5432 + args.port = args.port || 5432; + + // Default to host 127.0.0.1 + args.hostname = args.hostname || "127.0.0.1"; + + connection = net.createConnection(args.port, args.hostname); + events = new process.EventEmitter(); + query_queue = []; + readyState = false; + closeState = false; + started = false; + conn = this; + current_query = null; + + // Disable the idle timeout on the connection + connection.setTimeout(0); - // Set up callbacks to automatically do the login and other logic - events.addListener('AuthenticationMD5Password', function (salt) { - var result = "md5" + md5(md5(args.password + args.username) + salt.toString("binary")); - sendMessage('PasswordMessage', [result]); - }); - events.addListener('AuthenticationCleartextPassword', function () { - sendMessage('PasswordMessage', [args.password]); - }); - events.addListener('ErrorResponse', function (e) { - conn.emit('error', e.S + ": " + e.M); - if (e.S === 'FATAL') { - connection.end(); + // Sends a message to the postgres server + function sendMessage(type, args) { + var buffer = (formatter[type].apply(this, args)).frame(); + if (exports.DEBUG > 0) { + sys.debug("Sending " + type + ": " + JSON.stringify(args)); + if (exports.DEBUG > 2) { + sys.debug("->" + buffer.inspect().replace('<', '[')); + } + } + connection.write(buffer); } - }); - events.addListener('ReadyForQuery', function () { - if (!started) { - started = true; - conn.emit('connection'); + + var queue = []; + function checkInput() { + if (queue.length === 0) { return; } + var first = queue[0]; + var code = String.fromCharCode(first[0]); + var length = first.int32Read(1) - 4; + + // Make sure we have a whole message, TCP comes in chunks + if (first.length < length + 5) { + if (queue.length > 1) { + // Merge the first two buffers + queue.shift(); + var b = new Buffer(first.length + queue[0].length); + first.copy(b); + queue[0].copy(b, first.length); + queue[0] = b; + return checkInput(); + } else { + return; + } + } + // What does this do? + var message = first.slice(5, 5 + length); + if (first.length === 5 + length) { + queue.shift(); + } else { + queue[0] = first.slice(length + 5, first.length); + } + + if (exports.DEBUG > 1) { + sys.debug("stream: " + code + " " + message.inspect()); + } + // This shouldn't block. + // TODO: Rewrite into a callback. + command = parse_response(code, message); + if (command.type) { + if (exports.DEBUG > 0) { + sys.debug("Received " + command.type + ": " + JSON.stringify(command.args)); + } + command.args.unshift(command.type); + // Uses a selective emitter. + // First, tests whether or not the executing query listens to the event. + // This permits a given query to take over selected aspects of the + // If not, fires on the primary (connection) event loop. + + if (current_query.listeners(command.type) >= 1) { + current_query.emit.apply(current_query, command.args); + } + else { + events.emit.apply(events, command.args); + } + } + checkInput(); } - if (query_queue.length > 0) { - var query = query_queue.shift(); - query_callback = query.callback; - row_callback = query.row_callback; - sendMessage('Query', [query.sql]); - readyState = false; - } else { - if (closeState) { + + // Set up tcp client + connection.addListener("connect", function () { + sendMessage('StartupMessage', [{user: args.username, database: args.database}]); + }); + connection.addListener("data", function (data) { + if (exports.DEBUG > 2) { + sys.debug("<-" + data.inspect()); + } + queue.push(data); + checkInput(); + }); + connection.addListener("end", function (data) { connection.end(); - } else { - readyState = true; - } - } - }); - events.addListener("RowDescription", function (data) { - row_description = data; - results = []; - }); - events.addListener("DataRow", function (data) { - var row, i, l, description, value; - row = {}; - l = data.length; - for (i = 0; i < l; i += 1) { - description = row_description[i]; - value = data[i]; - if (value !== null) { - // TODO: investigate to see if these numbers are stable across databases or - // if we need to dynamically pull them from the pg_types table - switch (description.type_id) { - case 16: // bool - value = value === 't'; - break; - case 20: // int8 - case 21: // int2 - case 23: // int4 - value = parseInt(value, 10); - break; + }); + connection.addListener("disconnect", function (had_error) { + if (had_error) { + sys.debug("CONNECTION DIED WITH ERROR"); } - } - row[description.field] = value; - } - if (row_callback) { - row_callback(row); - } else { - results.push(row); - } - }); - events.addListener('CommandComplete', function (data) { - query_callback.call(this, results); - }); - - conn.execute = function (sql/*, *parameters*/) { - var parameters = Array.prototype.slice.call(arguments, 1); - var callback = parameters.pop(); - - // Merge the parameters in with the sql if needed. - sql = sqllib.merge(sql, parameters); + }); - // TODO: somehow give the query_queue a hint that this isn't query and it - // can optimize. - query_queue.push({sql: sql, callback: function () { - callback(); - }}); + // Set up callbacks to automatically do the login and other logic + events.addListener('AuthenticationMD5Password', function (salt) { + var result = "md5" + md5(md5(args.password + args.username) + salt.toString("binary")); + sendMessage('PasswordMessage', [result]); + }); + events.addListener('AuthenticationCleartextPassword', function () { + sendMessage('PasswordMessage', [args.password]); + }); + events.addListener('ErrorResponse', function (e) { + conn.emit('error', e.S + ": " + e.M); + if (e.S === 'FATAL') { + connection.end(); + } + }); - if (readyState) { - events.emit('ReadyForQuery'); - } + events.addListener('ReadyForQuery', function () { + if (!started) { + started = true; + conn.emit('connection'); + } + + if (query_queue.length > 0 && readyState != false) { + var query = query_queue.shift(); + + current_query = query; + + /* + query_callback = query.callback; + row_callback = query.row_callback; + + // Implicit assumption we're only putting queries on the wire. + sendMessage('Query', [query.sql]); + */ + + conn.emit("nextMessage"); + readyState = false; + + } /*else { + if (closeState) { + connection.end(); + } else { + readyState = true; + } + }*/ + }); + // This should always be caught by the current query. + events.addListener("RowDescription", function (data) { + row_description = data; + results = []; + }); - }; - - conn.query = function query(sql/*, *parameters, row_callback*/) { - var row_callback, parameters, callback; - - // Grab the variable length parameters and the row_callback is there is one. - parameters = Array.prototype.slice.call(arguments, 1); - callback = parameters.pop(); - if (typeof parameters[parameters.length - 1] === 'function') { - row_callback = parameters.pop(); - } - - // Merge the parameters in with the sql if needed. - if (parameters.length > 0) { - sql = sqllib.merge(sql, parameters); + // + + events.addListener("DataRow", function (data) { + var row, i, l, description, value; + row = {}; + l = data.length; + for (i = 0; i < l; i += 1) { + description = row_description[i]; + value = data[i]; + if (value !== null) { + // TODO: investigate to see if these numbers are stable across databases or + // if we need to dynamically pull them from the pg_types table + switch (description.type_id) { + case 16: // bool + value = value === 't'; + break; + case 20: // int8 + case 21: // int2 + case 23: // int4 + value = parseInt(value, 10); + break; + } + } + row[description.field] = value; + } + if (current_query.listeners("newRow") > 0) { + current_query.emit("newRow", row); + } + else { + results.push(row) + } + }); + + + events.addListener('CommandComplete', function (data) { + if (results.length >= 1) { + // To allow for insert..returning + current_query.emit("Complete", results, data); + results = []; // blank the current result buffer. + } + else { + // Send the typing information. + current_query.emit("Complete", data); + } + // query_callback.call(this, results); + }); + + // conn.execute = function (sql/*, *parameters*/) { + // var parameters = Array.prototype.slice.call(arguments, 1); + // var callback = parameters.pop(); + // + // // Merge the parameters in with the sql if needed. + // // sql = sqllib.merge(sql, parameters); + // + // // TODO: somehow give the query_queue a hint that this isn't query and it + // // can optimize. + // query_queue.push({sql: sql, callback: function () { + // callback(); + // }}); + // + // if (readyState) { + // events.emit('ReadyForQuery'); + // } + // }; + + + conn.query = function query(query) { + var parameters, callback; + + // Grab the variable length parameters and the row_callback is there is one. + parameters = Array.prototype.slice.call(arguments, 1); + + if (typeof parameters[parameters.length - 1] === 'function') { + callback = parameters.pop(); + } + var q; + if (parameters.length == 1 && parameters[0] instanceof Array) { + // We have a parameterized query + q = new Prepared(query, function (sth) { + sth.execute(parameters[0], callback); + }); + } + else { + q = new Query(query, callback); + + } + query_queue.push(q); + events.emit("queryAdded"); + + + // if (row_callback) { + // query_queue.push({sql: sql, row_callback: row_callback, callback: function () { + // callback(); + // }}); + // } else { + // query_queue.push({sql: sql, callback: function (data) { + // callback(data); + // }}); + // } + // + // if (readyState) { + // events.emit('ReadyForQuery'); + // } + }; + + conn.prepare = function prepare(query, callback) { + // Sets up a prepared query, and drops it onto the queue. + query_queue.push( + new Prepared(query, callback); + ); + events.emit("queryAdded"); + // conn.emit.call(conn, "queryAdded"); } - - if (row_callback) { - query_queue.push({sql: sql, row_callback: row_callback, callback: function () { - callback(); - }}); - } else { - query_queue.push({sql: sql, callback: function (data) { - callback(data); - }}); + + this.end = function () { + closeState = true; + + // Close the connection right away if there are no pending queries + if (readyState) { + connection.end(); + } + }; + + events.addListener("queryAdded", function () { + if (readyState) { + conn.emit("ReadyForQuery"); + } + }); + + /* Allows a currently-executing query to selectively modify the current + query queue, IF it is currently executing. + + Otherwise, it splices the query object in after its position in the queue. + */ + conn.yield_to = function (current, query) { + if (current === current_query) { + current_query = query; + conn.emit("nextMessage"); + } } - if (readyState) { - events.emit('ReadyForQuery'); + conn.release = function (query) { + if (query === current_query) { + readyState = true; + } } - }; - - this.end = function () { - closeState = true; - - // Close the connection right away if there are no pending queries - if (readyState) { - connection.end(); + // Pumps the current_query queue via .next() + // and runs whatever we get back. + events.addListener("nextMessage", function () { + var msg; + if (current_query != null) { + msg = current_query.next(); + if (msg.type && msg.args) { + // We have what we need to perform this query. + sendMessage.apply(conn, msg); + } + } + }); + + conn.next = function (query) { + if (query === current_query) { + events.emit("nextMessage"); + } + // If it's not, why are you doing this? } - }; + + /* + Keeps watch for the addition of listeners on our connection. + This allows for monitoring the driver for notification requests, which, + in turn, allows us to catch that the user wants this particular + notification from the DB watched for. + + Ergo, we set up a DB listener with the same name, and fire our emitter + when it's triggered. + + Easy. + */ + conn.addListener('newListener', function (e, listener) { + if (e === 'String') { + // It's a string. + if (!(e in ['newListener'])) + conn.notify(e, listener); + } + + }); } Connection.prototype = new process.EventEmitter(); + + +/* Connection.prototype.get_store = function (name, columns) { return new sqllib.Store(this, name, columns, { do_insert: function (data, keys, values, callback) { @@ -467,5 +761,5 @@ Connection.prototype.get_store = function (name, columns) { types: ["_id SERIAL"] }); }; - -exports.Connection = Connection; +*/ +exports.connect = Connection; From 2d8238125b856ae1287bc6e9de39b7162e74bf27 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Mon, 2 Aug 2010 09:35:44 -0700 Subject: [PATCH 16/54] Cleaned up the syntax errors; postgres-pure.js now compiles. Time to start debugging. --- lib/postgres-pure.js | 36 +++---- lib/sql.js | 244 ------------------------------------------- 2 files changed, 12 insertions(+), 268 deletions(-) delete mode 100644 lib/sql.js diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index ae9a8dc..eee9676 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -218,7 +218,7 @@ function parse_response(code, buffer) { } -function Query(sql) { +function Query(sql, callback) { this.sql = sql; var q = this; var row_description, results; @@ -246,7 +246,12 @@ function Query(sql) { q.addListener("RowDescription", function (desc) { row_description = desc; }); - q.addListener("newRow", function (row)) + q.addListener("newRow", function (row) { + results.push(row); + }); + q.addListener("Complete", function (data) { + callback(results); + }); } Query.prototype = new process.EventEmitter; @@ -256,8 +261,6 @@ function Prepared(sql, conn, callback) { var prepared_name = md5(sql); // Use the md5 hash. This is easily selectable later. var q = this; q.row_description = null; - arr.push(); - var arr = [ { type: "Parse", @@ -320,7 +323,7 @@ function Prepared(sql, conn, callback) { } } - q.next() { + q.next = function() { if (arr[pos + 1] !== null) { pos = pos + 1; return arr[pos]; @@ -328,9 +331,9 @@ function Prepared(sql, conn, callback) { } } -Prepare.prototype = new EventEmitter; +Prepared.prototype = new process.EventEmitter; -function execPrepared = function (portal, prepared, args, callback) { +function execPrepared (portal, prepared, args, callback) { var q = this; var results = []; var arr = [ @@ -472,7 +475,7 @@ function Connection(args) { // This permits a given query to take over selected aspects of the // If not, fires on the primary (connection) event loop. - if (current_query.listeners(command.type) >= 1) { + if (current_query != null && current_query.listeners(command.type) >= 1) { current_query.emit.apply(current_query, command.args); } else { @@ -641,27 +644,12 @@ function Connection(args) { } query_queue.push(q); events.emit("queryAdded"); - - - // if (row_callback) { - // query_queue.push({sql: sql, row_callback: row_callback, callback: function () { - // callback(); - // }}); - // } else { - // query_queue.push({sql: sql, callback: function (data) { - // callback(data); - // }}); - // } - // - // if (readyState) { - // events.emit('ReadyForQuery'); - // } }; conn.prepare = function prepare(query, callback) { // Sets up a prepared query, and drops it onto the queue. query_queue.push( - new Prepared(query, callback); + new Prepared(query, callback) ); events.emit("queryAdded"); // conn.emit.call(conn, "queryAdded"); diff --git a/lib/sql.js b/lib/sql.js deleted file mode 100644 index d2b338c..0000000 --- a/lib/sql.js +++ /dev/null @@ -1,244 +0,0 @@ -/* -Copyright (c) 2010 Tim Caswell - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -// Escape of values from native to SQL string. -function sql_escape(value) { - if (value === null) { - return "NULL"; - } - if (value === true) { - return "TRUE"; - } - if (value === false) { - return "FALSE"; - } - if (value.constructor.name === 'String') { - return "'" + value.replace("'", "''") + "'"; - } - return value.toString(); -} - -// Fill in the placeholders with native values -function merge(sql, parameters) { - if (parameters.length === 0) { - return sql; - } - if (parameters.length === 1 && parameters[0].constructor.name === 'Object') { - parameters = parameters[0]; - // Named parameters - for (var key in parameters) { - if (parameters.hasOwnProperty(key)) { - sql = sql.replace(":" + key, sql_escape(parameters[key])); - } - } - } else { - if (parameters.length === 1 && parameters[0].constructor.name === 'Array') { - parameters = parameters[0]; - } - // ordered parameters - parameters.forEach(function (param) { - sql = sql.replace("?", sql_escape(param)); - }); - } - return sql; -} - -// Converter between JS types and SQL data types -function js_to_sql(class) { - if (class === String) { - return 'text'; - } - if (class === Number) { - return 'integer'; - } - if (class === Boolean) { - return 'bool'; - } - throw "Unknown type " + class; -} - -// Convert a condition hash/array to a proper SQL where clause. -function condition_to_sql(condition, value) { - var operator, - p = condition.indexOf(' '); - if (p === -1) { - return condition + " = " + sql_escape(value); - } - operator = condition.substr(p + 1); - condition = condition.substr(0, p); - if (['<', '>', '=', '<=', '>=', '!=', '<>'].indexOf(operator) >= 0) { - return condition + " " + operator + " " + sql_escape(value); - } - if (operator === '%') { - return condition + " LIKE " + sql_escape(value); - } - sys.debug(operator); - throw "Invalid operator " + operator; -} - -// overrides needs to contain at least the following -// index_col: the name of the special index column rowid in sqlite and oid in postgres -// do_insert: function (data, keys, values, callback) -// do_update: function (data, pairs, callback) - -function Store(conn, name, columns, overrides) { - var key, - types = []; - - this.name = name; - this.conn = conn; - - if (overrides.types) { - types = overrides.types; - delete overrides.types; - } - - if (columns) { - for (key in columns) { - if (columns.hasOwnProperty(key)) { - types.push(key + " " + js_to_sql(columns[key])); - } - } - - conn.execute("CREATE TABLE " + name + "(" + types.join(", ") +")", function () {}); - } - - if (overrides) { - var self = this; - Object.keys(overrides).forEach(function (key) { - self[key] = overrides[key]; - }); - } - - -} -Store.prototype = { - - get: function (id, callback) { - this.conn.query( - "SELECT " + this.index_col + " AS _id, * FROM " + this.name + " WHERE " + this.index_col + " = ?", id, - function (data) { - callback(data[0]); - } - ); - }, - - find: function (conditions, row_callback, callback) { - // row_callback is optional - if (typeof callback === 'undefined') { - callback = row_callback; - row_callback = false; - } - var sql; - // Shortcut if there are no conditions. - if (conditions === undefined || conditions.length === 0) { - return this.all(callback); - } - - if (conditions.constructor.name !== 'Array') { - conditions = [conditions]; - } - - sql = "SELECT " + this.index_col + " AS _id, * FROM " + this.name + " WHERE " + - conditions.map(function (group) { - var ands = [], key; - for (key in group) { - if (group.hasOwnProperty(key)) { - ands.push(condition_to_sql(key, group[key])); - } - } - return "(" + ands.join(" AND ") + ")"; - }).join(" OR "); - - if (row_callback) { - this.conn.query(sql, row_callback, callback); - } - this.conn.query(sql, callback); - }, - - each: function (row_callback, callback) { - return this.conn.query("SELECT " + this.index_col + " AS _id, * FROM " + this.name, row_callback, callback); - }, - - all: function (callback) { - return this.conn.query("SELECT " + this.index_col + " AS _id, * FROM " + this.name, callback); - }, - - do_update: function (data, pairs, callback) { - this.conn.execute("UPDATE " + this.name + - " SET " + pairs.join(", ") + - " WHERE " + this.index_col + " = " + sql_escape(data._id), - function () { - callback(); - } - ); - }, - - // Save a data object to the database. If it already has an _id do an update. - save: function (data, callback) { - var keys = [], - values = [], - pairs = [], - key; - - if (data._id) { - for (key in data) { - if (data.hasOwnProperty(key) && key !== '_id') { - pairs.push(key + " = " + sql_escape(data[key])); - } - } - this.do_update(data, pairs, callback); - } else { - for (key in data) { - if (data.hasOwnProperty(key)) { - keys.push(key); - values.push(sql_escape(data[key])); - } - } - this.do_insert(data, keys, values, callback); - } - }, - - // Remove an entry from the database and remove the _id from the data object. - remove: function (data, callback) { - if (typeof data === 'number') { - data = {_id: data}; - } - this.conn.execute("DELETE FROM " + this.name + - " WHERE " + this.index_col + " = " + sql_escape(data._id), - function () { - delete data._id; - callback() - } - ); - }, - - nuke: function (callback) { - this.conn.query("DELETE FROM " + this.name, callback); - } - -}; - -exports.merge = merge; -exports.Store = Store; - - From 0f3401bcfb523af98026ff04843a2112026a06c6 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Mon, 9 Aug 2010 11:05:09 -0700 Subject: [PATCH 17/54] Updated postgres-pure.js to compile and begin to actually work. Modification to demo.js to provide correct syntax ($1). Currently having an issue with the DESCRIBE message; modified buffer_extras to try to solve it. No luck as yet. --- demo.js | 17 +++++---- lib/buffer_extras.js | 26 ++++++++----- lib/postgres-pure.js | 91 ++++++++++++++++++++++++++++++-------------- 3 files changed, 89 insertions(+), 45 deletions(-) diff --git a/demo.js b/demo.js index 24fdaf9..3873130 100644 --- a/demo.js +++ b/demo.js @@ -1,9 +1,12 @@ var sys = require("sys"); -var Postgres = require("./postgres"); +var pg = require("./lib/postgres-pure"); +pg.DEBUG=true; -var db = new Postgres.Connection("dbname", "username", "password"); -db.query("SELECT * FROM test"); -db.query("SELECT * FROM test", function (data) { - sys.p(data); -}); -db.close(); +var db = new pg.connect("pgsql://aurynn:12345@localhost:5432/akhyana"); +db.prepare("SELECT * FROM users where id = $1;", function (sth) { + sth.execute([1], function (rs) { + for (var i = 0; i <= rs.length; i++) { + sys.puts(rs[i]); + } + }); +}); \ No newline at end of file diff --git a/lib/buffer_extras.js b/lib/buffer_extras.js index ca4d372..29cf47e 100644 --- a/lib/buffer_extras.js +++ b/lib/buffer_extras.js @@ -87,22 +87,28 @@ Buffer.makeWriter = function makeWriter() { return writer; }, int16: function pushInt16(number) { - var b = new Buffer(2); - b.int16Write(number); - data.push(b); - return writer; + var b = new Buffer(2); + b.int16Write(number); + data.push(b); + return writer; }, string: function pushString(string) { - data.push(Buffer.fromString(string)); - return writer; + data.push(Buffer.fromString(string)); + return writer; + }, + byte1: function byte1(string) { + var b = new Buffer(1); // one byte. + b.write(string, 'ascii'); + data.push(b); + return writer; }, cstring: function pushCstring(string) { - data.push(Buffer.fromString(string + "\0")); - return writer; + data.push(Buffer.fromString(string + "\0")); + return writer; }, multicstring: function pushMulticstring(fields) { - data.push(Buffer.fromString(fields.join("\0") + "\0\0")); - return writer; + data.push(Buffer.fromString(fields.join("\0") + "\0\0")); + return writer; }, hash: function pushHash(hash) { var keys = Object.keys(hash); diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index eee9676..c151106 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -1,5 +1,6 @@ /* -Copyright (c) 2010 Tim Caswell +Copyright (c) 2010 Tim Caswell , + (c) 2010 Aurynn Shaw Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -24,7 +25,6 @@ THE SOFTWARE. var md5 = require('./md5'); var net = require("net"); var sys = require("sys"); -//var sqllib = require('./sql'); var url = require('url'); var Buffer = require('./buffer_extras'); @@ -59,7 +59,7 @@ var formatter = { }, Describe: function (name, type) { return (encoder('D')) - .push.string(type) + .push.byte1(type) // Byte string aka ascii. .push.cstring(name); }, Execute: function (name, max_rows) { @@ -78,9 +78,12 @@ var formatter = { .push.cstring(name) .push.cstring(query) .push.int16(var_types.length); - var_types.each(function (var_type) { - builder.push.int32(var_type); - }); + for (var i = 0; i < var_types.length; i++) { + builder.push.int32(var_types[i]); + } + // var_types.each(function (var_type) { + // builder.push.int32(var_type); + // }); return builder; }, PasswordMessage: function (password) { @@ -210,7 +213,12 @@ function parse_response(code, buffer) { args[0][field[0]] = field.substr(1); }); break; + case '1': + type = 'ParseComplete'; + args = [{}]; + break; } + if (!type) { sys.debug("Unknown response " + code); } @@ -261,6 +269,7 @@ function Prepared(sql, conn, callback) { var prepared_name = md5(sql); // Use the md5 hash. This is easily selectable later. var q = this; q.row_description = null; + var pos = 0; var arr = [ { type: "Parse", @@ -273,12 +282,10 @@ function Prepared(sql, conn, callback) { }, { type: "Flush", - args: ['S', prepared_name], + args: [], }, ]; // This describes a (nearly) complete lifecycle of a prepared statement. - - q.addListener("ParseComplete", function () { // Execute can now be run successfully. // Until this point, we can't assume that there's a matching query. @@ -286,7 +293,7 @@ function Prepared(sql, conn, callback) { // description in our object. // Later optimization might hold on to these objects as hashes in the // connection object. - conn.next(); + // conn.next(); }); q.addListener("RowDescription", function (args) { @@ -324,11 +331,11 @@ function Prepared(sql, conn, callback) { } q.next = function() { - if (arr[pos + 1] !== null) { + if (arr[pos] !== null) { pos = pos + 1; - return arr[pos]; + return arr[pos-1]; } - + return null; } } Prepared.prototype = new process.EventEmitter; @@ -376,7 +383,6 @@ function execPrepared (portal, prepared, args, callback) { this.type = data.type; callback.call(data); } - }); } @@ -420,6 +426,9 @@ function Connection(args) { // Sends a message to the postgres server function sendMessage(type, args) { + if (exports.DEBUG > 0 ) { + sys.debug("Got type of "+type) + } var buffer = (formatter[type].apply(this, args)).frame(); if (exports.DEBUG > 0) { sys.debug("Sending " + type + ": " + JSON.stringify(args)); @@ -428,6 +437,10 @@ function Connection(args) { } } connection.write(buffer); + if (current_query) { + conn.next(current_query); // We don't always expect to get a response message. + // And if we do, the message object can sort it out. + } } var queue = []; @@ -521,14 +534,27 @@ function Connection(args) { }); events.addListener('ReadyForQuery', function () { + if (exports.DEBUG > 0) { + sys.debug("In RFQ"); + } if (!started) { started = true; conn.emit('connection'); } - if (query_queue.length > 0 && readyState != false) { - var query = query_queue.shift(); + if (closeState) { + connection.end(); + } else { + readyState = true; + } + if (exports.DEBUG > 0) { + sys.debug(readyState); + sys.debug("Queue length: "+query_queue.length); + } + + if (query_queue.length > 0 && readyState !== false) { + var query = query_queue.shift(); current_query = query; /* @@ -539,16 +565,10 @@ function Connection(args) { sendMessage('Query', [query.sql]); */ - conn.emit("nextMessage"); readyState = false; + events.emit("nextMessage"); - } /*else { - if (closeState) { - connection.end(); - } else { - readyState = true; - } - }*/ + } }); // This should always be caught by the current query. events.addListener("RowDescription", function (data) { @@ -678,7 +698,7 @@ function Connection(args) { conn.yield_to = function (current, query) { if (current === current_query) { current_query = query; - conn.emit("nextMessage"); + events.emit("nextMessage"); } } @@ -691,12 +711,27 @@ function Connection(args) { // Pumps the current_query queue via .next() // and runs whatever we get back. events.addListener("nextMessage", function () { + + if (events.DEBUG > 0) { + sys.debug("got nextMessage"); + } + var msg; - if (current_query != null) { + if (current_query !== null) { + + if (exports.DEBUG > 0) { + sys.debug("current query is not Null."); + } + msg = current_query.next(); - if (msg.type && msg.args) { + + if (exports.DEBUG > 0) { + sys.debug("Message is: "+msg); + } + + if (msg !== undefined && msg.type && msg.args) { // We have what we need to perform this query. - sendMessage.apply(conn, msg); + sendMessage.apply(conn, [msg.type, msg.args]); } } }); From 8037fcf50216b35553230461c0871759f3cda0d2 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Tue, 10 Aug 2010 07:46:22 -0700 Subject: [PATCH 18/54] Modification of the Describe message; switched the order of the arguments. --- lib/buffer_extras.js | 4 ++-- lib/postgres-pure.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/buffer_extras.js b/lib/buffer_extras.js index 29cf47e..dc84e23 100644 --- a/lib/buffer_extras.js +++ b/lib/buffer_extras.js @@ -97,8 +97,8 @@ Buffer.makeWriter = function makeWriter() { return writer; }, byte1: function byte1(string) { - var b = new Buffer(1); // one byte. - b.write(string, 'ascii'); + var b = new Buffer(1); // one octet. + b.write(string, 'ascii', 0); data.push(b); return writer; }, diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index c151106..37b013e 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -57,7 +57,7 @@ var formatter = { CopyDone: function () { // TODO: implement }, - Describe: function (name, type) { + Describe: function (type, name) { return (encoder('D')) .push.byte1(type) // Byte string aka ascii. .push.cstring(name); @@ -278,7 +278,7 @@ function Prepared(sql, conn, callback) { }, { type: "Describe", - args: ['S', prepared_name], + args: ["S", prepared_name], }, { type: "Flush", From 3724aae1cdaa4263a0094f8d18b05d8ef7acbdfa Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Tue, 17 Aug 2010 21:02:08 -0700 Subject: [PATCH 19/54] Added the "Bind" message. Modification to the Prepare blocks so they actually work. Prepare now yields more or less correctly to the Execute message(s) Bind message isn't working, and the code needs a lot of optimization and thought before it can be considered production-usable --- demo.js | 3 +- lib/postgres-pure.js | 171 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 142 insertions(+), 32 deletions(-) diff --git a/demo.js b/demo.js index 3873130..1d126cf 100644 --- a/demo.js +++ b/demo.js @@ -3,7 +3,8 @@ var pg = require("./lib/postgres-pure"); pg.DEBUG=true; var db = new pg.connect("pgsql://aurynn:12345@localhost:5432/akhyana"); -db.prepare("SELECT * FROM users where id = $1;", function (sth) { +db.prepare("SELECT * FROM users WHERE id = $1;", function (sth) { + sys.debug("in callback"); sth.execute([1], function (rs) { for (var i = 0; i <= rs.length; i++) { sys.puts(rs[i]); diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 37b013e..17d62bf 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -51,6 +51,33 @@ function encoder(header) { // http://www.postgresql.org/docs/8.3/static/protocol-message-formats.html var formatter = { + Bind: function (portal, prepared_name, args) { + var builder = (encoder('B')) + .push.cstring(portal) + .push.cstring(prepared_name) + .push.int16(0) // 0 format codes. We don't want to declare types. + .push.int16(args.length); + for (var i = 0; i < args.length; i++) { + switch (typeof args[i]) { + case "number": + builder.push.int32(4); // 4 bytes. int32. + builder.push.int32(args[i]); + break; + case "string": + builder.push.int32(args[i].length); + builder.push.string(args[i]); // Not a cstring. Don't \0 + break; + case "boolean": + builder.push.int32(1); // One byte. + builder.push.string(args[i] ? 1 : 0); + break; + } + } + builder.push.int16(0); // They should all use text. Don't declare return + // types, as we already have the types from the + // ParameterDescription + return builder; + }, CopyData: function () { // TODO: implement }, @@ -159,6 +186,15 @@ function parse_response(code, buffer) { args[0][field[0]] = field.substr(1); }); break; + case 't': + type = "ParameterDescription", + num_fields = reader.int16(); + data = []; + for (var i = 0; i < num_fields; i++) { + data.push(reader.int32()); + } + args = data; + break; case 'S': type = "ParameterStatus"; args = [reader.cstring(), reader.cstring()]; @@ -175,7 +211,7 @@ function parse_response(code, buffer) { type = "RowDescription"; num_fields = reader.int16(); data = []; - for (i = 0; i < num_fields; i += 1) { + for (var i = 0; i < num_fields; i += 1) { data.push({ field: reader.cstring(), table_id: reader.int32(), @@ -217,6 +253,10 @@ function parse_response(code, buffer) { type = 'ParseComplete'; args = [{}]; break; + case 'n': + type = 'NoData'; + args = []; + break; } if (!type) { @@ -268,8 +308,17 @@ function Prepared(sql, conn, callback) { var prepared_name = md5(sql); // Use the md5 hash. This is easily selectable later. var q = this; + var portal = 'meh'; q.row_description = null; + q.param_description = null; + q.noData = false; + + var parseComplete = null; + var readyToExec = false; + var conn = conn; + var pos = 0; + var callback = callback; var arr = [ { type: "Parse", @@ -294,12 +343,54 @@ function Prepared(sql, conn, callback) { // Later optimization might hold on to these objects as hashes in the // connection object. // conn.next(); + parseComplete = true; + q.emit("executable?"); }); q.addListener("RowDescription", function (args) { q.row_description = args; + if (exports.DEBUG > 0) { + sys.debug("Caught RowDescription in Prepared object."); + } + q.emit("executable?"); + }); + + var execute = []; + + q.addListener("executable?", function () { + + if (exports.DEBUG > 0) { + sys.debug("hit executable?"); + sys.debug("parseComplete: "+parseComplete); + sys.debug("readyToExec: " + readyToExec); + sys.debug((q.row_description != null && q.row_description.length > 0) || q.noData); + } + + if (parseComplete && readyToExec && ((q.row_description != null && q.row_description.length > 0) || q.noData)) { + conn.release(q); // Yield myself. + if (exports.DEBUG > 0) { + sys.debug("Yielding to the next message set."); + } + } + else if (parseComplete && ((q.row_description != null && q.row_description.length > 0) || q.noData)) { + // The server is prepped. We can now safely run the callback + // and trap the execute statements, prior to adding them to the + // main query queue. + sys.debug("calling callback"); + callback(q); + q.emit("executable?"); + } + }); + + q.addListener("ParameterDescription", function (desc) { + q.param_description = desc; + q.emit("executable?"); }); + q.addListener("NoData", function () { + q.noData = true; + }) + /* Executing the function tests whether or not we've been passed an argument list. @@ -309,6 +400,8 @@ function Prepared(sql, conn, callback) { q.execute = function () { // If the first argument is an array, then we use that as our bind // parameters. Otherwise, arguments[0] should be a function. + readyToExec = true; + var eP; if (arguments[0] instanceof Array) { if (arguments[0].length >= 1) { var callback = null; @@ -316,18 +409,22 @@ function Prepared(sql, conn, callback) { if (typeof(arguments[1]) == 'function') { callback = arguments[1]; } + eP = new execPrepared(portal, prepared_name, arguments[0], arguments[1]); } } else if (typeof(arguments[0]) == 'function' ) { - var eP = new execPrepared(portal, prepared_name, [], arguments[0]); + eP = new execPrepared(portal, prepared_name, [], arguments[0]); // Announcing that my slot is released in favour of this new query. // The connection doesn't advance the buffer, but *does* replace // the current query with this one. - conn.yield_to(eP); } else { q.emit("error", "First argument must be array or function!"); } + if(exports.DEBUG > 0) { + sys.debug("Yielding"); + } + conn.yield_to(q, eP); } q.next = function() { @@ -384,6 +481,14 @@ function execPrepared (portal, prepared, args, callback) { callback.call(data); } }); + var pos = 0; + q.next = function() { + if (arr[pos] !== null) { + pos = pos + 1; + return arr[pos-1]; + } + return null; + } } execPrepared.prototype = new process.EventEmitter; @@ -487,11 +592,22 @@ function Connection(args) { // First, tests whether or not the executing query listens to the event. // This permits a given query to take over selected aspects of the // If not, fires on the primary (connection) event loop. - - if (current_query != null && current_query.listeners(command.type) >= 1) { + if (exports.DEBUG > 0) { + sys.debug("current_query is null: "+ current_query !== null); + if (current_query !== null) { + sys.debug("current_query listeners: " + current_query.listeners(command.type).length); + } + } + if (current_query !== null && current_query.listeners(command.type).length >= 1) { + if (exports.DEBUG > 0) { + sys.debug("Sending "+command.type+" to current_query"); + } current_query.emit.apply(current_query, command.args); } else { + if (exports.DEBUG > 0) { + sys.debug("Sending "+command.type+" to local handler"); + } events.emit.apply(events, command.args); } } @@ -576,14 +692,14 @@ function Connection(args) { results = []; }); - // + // This is no longer correct. Sigh. events.addListener("DataRow", function (data) { var row, i, l, description, value; row = {}; l = data.length; for (i = 0; i < l; i += 1) { - description = row_description[i]; + description = current_query.row_description[i]; value = data[i]; if (value !== null) { // TODO: investigate to see if these numbers are stable across databases or @@ -623,25 +739,6 @@ function Connection(args) { // query_callback.call(this, results); }); - // conn.execute = function (sql/*, *parameters*/) { - // var parameters = Array.prototype.slice.call(arguments, 1); - // var callback = parameters.pop(); - // - // // Merge the parameters in with the sql if needed. - // // sql = sqllib.merge(sql, parameters); - // - // // TODO: somehow give the query_queue a hint that this isn't query and it - // // can optimize. - // query_queue.push({sql: sql, callback: function () { - // callback(); - // }}); - // - // if (readyState) { - // events.emit('ReadyForQuery'); - // } - // }; - - conn.query = function query(query) { var parameters, callback; @@ -669,7 +766,7 @@ function Connection(args) { conn.prepare = function prepare(query, callback) { // Sets up a prepared query, and drops it onto the queue. query_queue.push( - new Prepared(query, callback) + new Prepared(query, conn, callback) ); events.emit("queryAdded"); // conn.emit.call(conn, "queryAdded"); @@ -695,16 +792,28 @@ function Connection(args) { Otherwise, it splices the query object in after its position in the queue. */ - conn.yield_to = function (current, query) { - if (current === current_query) { - current_query = query; - events.emit("nextMessage"); + conn.yield_to = function (first, query) { + if (exports.DEBUG > 0) { + sys.debug("got yield_to"); + } + if (first === current_query) { + query_queue.unshift(query); + events.emit("queryAdded"); // Don't immediately switch to the next message. + } + else if (first in query_queue) { + // Splice it in after the query + query_queue.splice( query_queue.indexOf(first), 0, query ); + events.emit("queryAdded"); } } conn.release = function (query) { + if (exports.DEBUG > 0) { + sys.debug("got release"); + } if (query === current_query) { readyState = true; + events.emit("ReadyForQuery"); // Cycle along. } } From ff835890bf09022ed3136dcce438230195504c02 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Tue, 24 Aug 2010 08:15:33 -0700 Subject: [PATCH 20/54] Prepared queries now work mostly as expected. :) --- demo.js | 12 ++++++-- lib/buffer_extras.js | 2 +- lib/postgres-pure.js | 70 ++++++++++++++++++++++++++++++++------------ 3 files changed, 62 insertions(+), 22 deletions(-) diff --git a/demo.js b/demo.js index 1d126cf..41f5b28 100644 --- a/demo.js +++ b/demo.js @@ -1,13 +1,19 @@ var sys = require("sys"); var pg = require("./lib/postgres-pure"); -pg.DEBUG=true; +pg.DEBUG=0; var db = new pg.connect("pgsql://aurynn:12345@localhost:5432/akhyana"); db.prepare("SELECT * FROM users WHERE id = $1;", function (sth) { sys.debug("in callback"); sth.execute([1], function (rs) { - for (var i = 0; i <= rs.length; i++) { - sys.puts(rs[i]); + sys.debug("In execute"); + for (var i = 0; i < rs.length; i++) { + for (var key in rs[i]) { + if (rs[i].hasOwnProperty(key)) { + sys.puts(key +": " +rs[i][key]); + } + } } + db.end(); }); }); \ No newline at end of file diff --git a/lib/buffer_extras.js b/lib/buffer_extras.js index dc84e23..8105b72 100644 --- a/lib/buffer_extras.js +++ b/lib/buffer_extras.js @@ -98,7 +98,7 @@ Buffer.makeWriter = function makeWriter() { }, byte1: function byte1(string) { var b = new Buffer(1); // one octet. - b.write(string, 'ascii', 0); + b.write(string, 'utf8', 0); data.push(b); return writer; }, diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 17d62bf..6527998 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -51,27 +51,45 @@ function encoder(header) { // http://www.postgresql.org/docs/8.3/static/protocol-message-formats.html var formatter = { - Bind: function (portal, prepared_name, args) { + Bind: function (portal, prepared_name, params, args) { var builder = (encoder('B')) .push.cstring(portal) .push.cstring(prepared_name) - .push.int16(0) // 0 format codes. We don't want to declare types. - .push.int16(args.length); + .push.int16(args.length); // declare our format codes as expected. + for (var i = 0; i < args.length; i++) { switch (typeof args[i]) { case "number": - builder.push.int32(4); // 4 bytes. int32. - builder.push.int32(args[i]); + case "boolean": + case "object": + builder.push.int16(1); // binary break; case "string": - builder.push.int32(args[i].length); - builder.push.string(args[i]); // Not a cstring. Don't \0 + builder.push.int16(0); + break; + } + } + + builder.push.int16(args.length); + for (var i = 0; i < args.length; i++) { + switch (typeof args[i]) { + case "number": + builder.push.int32(4) // 4 bytes. int32. + .push.int32(args[i]); + break; + case "string": + builder.push.int32(args[i].length) + .push.string(args[i]); // Not a cstring. Don't \0 break; case "boolean": - builder.push.int32(1); // One byte. - builder.push.string(args[i] ? 1 : 0); + builder.push.int32(1) // One byte. + .push.string(args[i] ? 1 : 0); break; - } + case "object": + if (args[i] === null) { + builder.push.int32(-1); + } + }; } builder.push.int16(0); // They should all use text. Don't declare return // types, as we already have the types from the @@ -193,7 +211,7 @@ function parse_response(code, buffer) { for (var i = 0; i < num_fields; i++) { data.push(reader.int32()); } - args = data; + args = [data]; break; case 'S': type = "ParameterStatus"; @@ -409,11 +427,11 @@ function Prepared(sql, conn, callback) { if (typeof(arguments[1]) == 'function') { callback = arguments[1]; } - eP = new execPrepared(portal, prepared_name, arguments[0], arguments[1]); + eP = new execPrepared(portal, prepared_name, q.param_description, arguments[0], q.row_description, conn, arguments[1]); } } else if (typeof(arguments[0]) == 'function' ) { - eP = new execPrepared(portal, prepared_name, [], arguments[0]); + eP = new execPrepared(portal, prepared_name, q.param_description, [], q.row_description, conn, arguments[0]); // Announcing that my slot is released in favour of this new query. // The connection doesn't advance the buffer, but *does* replace // the current query with this one. @@ -437,20 +455,25 @@ function Prepared(sql, conn, callback) { } Prepared.prototype = new process.EventEmitter; -function execPrepared (portal, prepared, args, callback) { +function execPrepared (portal, prepared, params, args, row_desc, conn, callback) { var q = this; + q.row_description = row_desc; var results = []; var arr = [ { type: "Execute", args: [portal, 0], // No limit. Get all the rows. + }, + { + type: "Flush", + args: [portal, 0] } ]; // If we have args, unshift the Bind. if (args instanceof Array && args.length >= 1) { arr.unshift({ type: "Bind", - args:[portal, prepared, args], + args:[portal, prepared, params, args], callback: callback }); } @@ -473,13 +496,20 @@ function execPrepared (portal, prepared, args, callback) { // If it was a SELECT, args will be an array of rows, // If it was an INSERT, etc, it'll be the type, and the number of // affected rows. + if (exports.DEBUG > 0) { + sys.debug("Results length " + results.length); + } if (results.length >= 1) { - callback.call(data, results); + if (exports.DEBUG > 0) { + sys.debug("Calling with results"); + } + callback.call(q, results); } else { this.type = data.type; - callback.call(data); + callback.call(q); } + conn.release(q); }); var pos = 0; q.next = function() { @@ -717,7 +747,11 @@ function Connection(args) { } row[description.field] = value; } - if (current_query.listeners("newRow") > 0) { + if (exports.DEBUG > 0) { + sys.debug(current_query.listeners("newRow").length); + sys.debug(current_query); + } + if (current_query.listeners("newRow").length > 0) { current_query.emit("newRow", row); } else { From 6922cf8ad46e4d2b3659ea52822f686496007000 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Wed, 25 Aug 2010 16:14:17 -0700 Subject: [PATCH 21/54] Adding in support for ? -> $1, etc. syntax. Very simple regex thus far, and will likely prove to be brittle. --- demo.js | 8 +++----- lib/postgres-pure.js | 4 ++++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/demo.js b/demo.js index 41f5b28..d6de0db 100644 --- a/demo.js +++ b/demo.js @@ -1,12 +1,10 @@ var sys = require("sys"); var pg = require("./lib/postgres-pure"); -pg.DEBUG=0; +pg.DEBUG=1; -var db = new pg.connect("pgsql://aurynn:12345@localhost:5432/akhyana"); -db.prepare("SELECT * FROM users WHERE id = $1;", function (sth) { - sys.debug("in callback"); +var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); +db.prepare("SELECT ?::int", function (sth) { sth.execute([1], function (rs) { - sys.debug("In execute"); for (var i = 0; i < rs.length; i++) { for (var key in rs[i]) { if (rs[i].hasOwnProperty(key)) { diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 6527998..2ac1adc 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -799,6 +799,10 @@ function Connection(args) { conn.prepare = function prepare(query, callback) { // Sets up a prepared query, and drops it onto the queue. + if (query.match(/\?/)) { + var i = 1; + query = query.replace(/\?/g, function () { return "$" + i++; }); + } query_queue.push( new Prepared(query, conn, callback) ); From 38ed09d1f631e42e2261e7c3a7d1984c0146e372 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Wed, 25 Aug 2010 16:24:27 -0700 Subject: [PATCH 22/54] Debugging cleanup. --- demo.js | 2 +- lib/postgres-pure.js | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/demo.js b/demo.js index d6de0db..b92470c 100644 --- a/demo.js +++ b/demo.js @@ -1,6 +1,6 @@ var sys = require("sys"); var pg = require("./lib/postgres-pure"); -pg.DEBUG=1; +pg.DEBUG=0; var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); db.prepare("SELECT ?::int", function (sth) { diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 2ac1adc..47ed124 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -278,7 +278,7 @@ function parse_response(code, buffer) { } if (!type) { - sys.debug("Unknown response " + code); + sys.debug("Unknown response " + code); } return {type: type, args: args}; } @@ -394,7 +394,9 @@ function Prepared(sql, conn, callback) { // The server is prepped. We can now safely run the callback // and trap the execute statements, prior to adding them to the // main query queue. - sys.debug("calling callback"); + if (exports.DEBUG > 0) { + sys.debug("calling callback"); + } callback(q); q.emit("executable?"); } From 98cfd120139d2fa169ce4d79a3c86b194f4f417e Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Mon, 30 Aug 2010 21:26:28 -0700 Subject: [PATCH 23/54] Adding the BindComplete message. Verified node 0.2.0 support. --- lib/postgres-pure.js | 159 ++++++++++++++++++++++--------------------- 1 file changed, 82 insertions(+), 77 deletions(-) diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 47ed124..9721cff 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -197,85 +197,90 @@ function parse_response(code, buffer) { break; } break; - case 'E': - type = "ErrorResponse"; - args = [{}]; - reader.multicstring().forEach(function (field) { - args[0][field[0]] = field.substr(1); - }); - break; - case 't': - type = "ParameterDescription", - num_fields = reader.int16(); - data = []; - for (var i = 0; i < num_fields; i++) { - data.push(reader.int32()); - } - args = [data]; - break; - case 'S': - type = "ParameterStatus"; - args = [reader.cstring(), reader.cstring()]; - break; - case 'K': - type = "BackendKeyData"; - args = [reader.int32(), reader.int32()]; - break; - case 'Z': - type = "ReadyForQuery"; - args = [reader.string(1)]; - break; - case 'T': - type = "RowDescription"; - num_fields = reader.int16(); - data = []; - for (var i = 0; i < num_fields; i += 1) { - data.push({ - field: reader.cstring(), - table_id: reader.int32(), - column_id: reader.int16(), - type_id: reader.int32(), - type_size: reader.int16(), - type_modifier: reader.int32(), - format_code: reader.int16() + case 'E': + type = "ErrorResponse"; + args = [{}]; + reader.multicstring().forEach(function (field) { + args[0][field[0]] = field.substr(1); }); - } - args = [data]; - break; - case 'D': - type = "DataRow"; - data = []; - num_fields = reader.int16(); - for (i = 0; i < num_fields; i += 1) { - size = reader.int32(); - if (size === -1) { - data.push(null); - } else { - data.push(reader.string(size)); - } - } - args = [data]; - break; - case 'C': - type = "CommandComplete"; - args = [reader.cstring()]; - break; - case 'N': - type = "NoticeResponse"; - args = [{}]; - reader.multicstring().forEach(function (field) { - args[0][field[0]] = field.substr(1); - }); - break; - case '1': - type = 'ParseComplete'; - args = [{}]; - break; - case 'n': - type = 'NoData'; - args = []; - break; + break; + case 't': + type = "ParameterDescription", + num_fields = reader.int16(); + data = []; + for (var i = 0; i < num_fields; i++) { + data.push(reader.int32()); + } + args = [data]; + break; + case 'S': + type = "ParameterStatus"; + args = [reader.cstring(), reader.cstring()]; + break; + case 'K': + type = "BackendKeyData"; + args = [reader.int32(), reader.int32()]; + break; + case 'Z': + type = "ReadyForQuery"; + args = [reader.string(1)]; + break; + case 'T': + type = "RowDescription"; + num_fields = reader.int16(); + data = []; + for (var i = 0; i < num_fields; i += 1) { + data.push({ + field: reader.cstring(), + table_id: reader.int32(), + column_id: reader.int16(), + type_id: reader.int32(), + type_size: reader.int16(), + type_modifier: reader.int32(), + format_code: reader.int16() + }); + } + args = [data]; + break; + case 'D': + type = "DataRow"; + data = []; + num_fields = reader.int16(); + for (i = 0; i < num_fields; i += 1) { + size = reader.int32(); + if (size === -1) { + data.push(null); + } else { + data.push(reader.string(size)); + } + } + args = [data]; + break; + case 'C': + type = "CommandComplete"; + args = [reader.cstring()]; + break; + case 'N': + type = "NoticeResponse"; + args = [{}]; + reader.multicstring().forEach(function (field) { + args[0][field[0]] = field.substr(1); + }); + break; + case '1': + type = 'ParseComplete'; + args = [{}]; + break; + case 'n': + type = 'NoData'; + args = []; + break; + case '2': + type = "BindComplete"; + args = [{}]; + break; } + if (!type) { sys.debug("Unknown response " + code); From 13fe013a26de6efa93a8f0c47433aeb1fcc31d26 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Mon, 30 Aug 2010 21:46:13 -0700 Subject: [PATCH 24/54] Cleanup of readme. Adding license to readme. Modification of the execute stage of prepared statements to allow multiple bound portals on the same prepared query. --- README.md | 22 +++++++++------------- demo.js | 3 +++ lib/postgres-pure.js | 31 ++++++------------------------- 3 files changed, 18 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 6f914ec..f818889 100644 --- a/README.md +++ b/README.md @@ -3,18 +3,18 @@ This library is a implementation of the PostgreSQL backend/frontend protocol in javascript. It uses the node.js tcp and event libraries. A javascript md5 library is included for servers that require md5 password hashing (this is default). -This library allows for the handling of prepared queries. - -Parameterized queries are (currently) no longer supported. +This library allows for the correct handling of prepared queries. If you wish to nest DB calls, db.close must be in the deepest callback, or all statements that occur inside of callbacks deeper than the callback which handles db.close will not be executed. +All code is available under the terms of the MIT license, unless otherwise noted (md5.js) + ## Example use var sys = require("sys"); - var Postgres = require("postgres"); + var pg = require("postgres"); - var db = new Postgres.Connection("database", "username", "password"); + var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); db.query("SELECT * FROM sometable", function (data) { sys.p(data); }); @@ -25,9 +25,8 @@ If you wish to nest DB calls, db.close must be in the deepest callback, or all s var sys = require("sys"); var pg = require("postgres"); - var db = new pg.Connection("database", "username", "password"); + var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); db.query("SELECT * FROM yourtable WHERE id = ?", [1], function (data) { - sys.p(data); }); db.close(); @@ -37,14 +36,11 @@ If you wish to nest DB calls, db.close must be in the deepest callback, or all s var sys = require("sys"); var pg = require("postgres"); - var db = new pg.Connection("database", "username", "password"); + var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); var stmt = db.prepare("SELECT * FROM yourtable WHERE id = ?"); - stmt.execute([1]).addCallback(function (d) { - + stmt.execute([1], function (d) { sys.p(d); db.close(); - }); - - + }); \ No newline at end of file diff --git a/demo.js b/demo.js index b92470c..8bdde6c 100644 --- a/demo.js +++ b/demo.js @@ -14,4 +14,7 @@ db.prepare("SELECT ?::int", function (sth) { } db.end(); }); + sth.execute([2], function (rs) { + sys.puts(sys.inspect(rs)); + }) }); \ No newline at end of file diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 9721cff..65c23b8 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -49,7 +49,7 @@ function encoder(header) { return w; } -// http://www.postgresql.org/docs/8.3/static/protocol-message-formats.html +// http://www.postgresql.org/docs/8.4/static/protocol-message-formats.html var formatter = { Bind: function (portal, prepared_name, params, args) { var builder = (encoder('B')) @@ -331,7 +331,6 @@ function Prepared(sql, conn, callback) { var prepared_name = md5(sql); // Use the md5 hash. This is easily selectable later. var q = this; - var portal = 'meh'; q.row_description = null; q.param_description = null; q.noData = false; @@ -434,11 +433,11 @@ function Prepared(sql, conn, callback) { if (typeof(arguments[1]) == 'function') { callback = arguments[1]; } - eP = new execPrepared(portal, prepared_name, q.param_description, arguments[0], q.row_description, conn, arguments[1]); + eP = new execPrepared(prepared_name, q.param_description, arguments[0], q.row_description, conn, arguments[1]); } } else if (typeof(arguments[0]) == 'function' ) { - eP = new execPrepared(portal, prepared_name, q.param_description, [], q.row_description, conn, arguments[0]); + eP = new execPrepared(prepared_name, q.param_description, [], q.row_description, conn, arguments[0]); // Announcing that my slot is released in favour of this new query. // The connection doesn't advance the buffer, but *does* replace // the current query with this one. @@ -462,8 +461,9 @@ function Prepared(sql, conn, callback) { } Prepared.prototype = new process.EventEmitter; -function execPrepared (portal, prepared, params, args, row_desc, conn, callback) { +function execPrepared (prepared, params, args, row_desc, conn, callback) { var q = this; + var portal = md5(prepared + args.join("|")); // Clearly need a generated portal name here. q.row_description = row_desc; var results = []; var arr = [ @@ -741,6 +741,7 @@ function Connection(args) { if (value !== null) { // TODO: investigate to see if these numbers are stable across databases or // if we need to dynamically pull them from the pg_types table + // TODO: Investigate javascript Date objects. switch (description.type_id) { case 16: // bool value = value === 't'; @@ -919,24 +920,4 @@ function Connection(args) { } Connection.prototype = new process.EventEmitter(); - -/* -Connection.prototype.get_store = function (name, columns) { - return new sqllib.Store(this, name, columns, { - do_insert: function (data, keys, values, callback) { - this.conn.query("INSERT INTO " + - this.name + "(" + keys.join(", ") + ")" + - " VALUES (" + values.join(", ") + ")" + - " RETURNING _id", - function (result) { - data._id = parseInt(result[0]._id, 10); - callback(data._id); - } - ); - }, - index_col: '_id', - types: ["_id SERIAL"] - }); -}; -*/ exports.connect = Connection; From 20752af3968f4223b1dddc59320c349799336170 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Mon, 13 Sep 2010 22:04:19 -0700 Subject: [PATCH 25/54] Breaking out the internal handling into transaction objects that can be put directly onto connection queues, as opposed to manipulating the wire directly. --- lib/postgres-pure.js | 452 +++++++++++++++++++++++++++---------------- 1 file changed, 288 insertions(+), 164 deletions(-) diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 65c23b8..a95a4d3 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -159,146 +159,169 @@ var formatter = { // Parse response streams from the server function parse_response(code, buffer) { - var input, type, args, num_fields, data, size, i; - reader = buffer.toReader(); - args = []; - switch (code) { - case 'R': - switch (reader.int32()) { - case 0: - type = "AuthenticationOk"; - break; - case 2: - type = "AuthenticationKerberosV5"; - break; - case 3: - type = "AuthenticationCleartextPassword"; - break; - case 4: - type = "AuthenticationCryptPassword"; - args = [reader.string(2)]; - break; - case 5: - type = "AuthenticationMD5Password"; - args = [reader.buffer(4)]; - break; - case 6: - type = "AuthenticationSCMCredential"; - break; - case 7: - type = "AuthenticationGSS"; - break; - case 8: - // TODO: add in AuthenticationGSSContinue - type = "AuthenticationSSPI"; - break; - default: - - break; - } - break; - case 'E': - type = "ErrorResponse"; - args = [{}]; - reader.multicstring().forEach(function (field) { - args[0][field[0]] = field.substr(1); - }); - break; - case 't': - type = "ParameterDescription", - num_fields = reader.int16(); - data = []; - for (var i = 0; i < num_fields; i++) { - data.push(reader.int32()); - } - args = [data]; - break; - case 'S': - type = "ParameterStatus"; - args = [reader.cstring(), reader.cstring()]; - break; - case 'K': - type = "BackendKeyData"; - args = [reader.int32(), reader.int32()]; - break; - case 'Z': - type = "ReadyForQuery"; - args = [reader.string(1)]; + var input, type, args, num_fields, data, size, i; + reader = buffer.toReader(); + args = []; + switch (code) { + case 'R': + switch (reader.int32()) { + case 0: + type = "AuthenticationOk"; + break; + case 2: + type = "AuthenticationKerberosV5"; + break; + case 3: + type = "AuthenticationCleartextPassword"; + break; + case 4: + type = "AuthenticationCryptPassword"; + args = [reader.string(2)]; + break; + case 5: + type = "AuthenticationMD5Password"; + args = [reader.buffer(4)]; + break; + case 6: + type = "AuthenticationSCMCredential"; + break; + case 7: + type = "AuthenticationGSS"; + break; + case 8: + // TODO: add in AuthenticationGSSContinue + type = "AuthenticationSSPI"; + break; + default: + break; + } break; - case 'T': - type = "RowDescription"; - num_fields = reader.int16(); - data = []; - for (var i = 0; i < num_fields; i += 1) { - data.push({ - field: reader.cstring(), - table_id: reader.int32(), - column_id: reader.int16(), - type_id: reader.int32(), - type_size: reader.int16(), - type_modifier: reader.int32(), - format_code: reader.int16() + case 'E': + type = "ErrorResponse"; + args = [{}]; + reader.multicstring().forEach(function (field) { + args[0][field[0]] = field.substr(1); }); + break; + case 't': + type = "ParameterDescription", + num_fields = reader.int16(); + data = []; + for (var i = 0; i < num_fields; i++) { + data.push(reader.int32()); + } + args = [data]; + break; + case 'S': + type = "ParameterStatus"; + args = [reader.cstring(), reader.cstring()]; + break; + case 'K': + type = "BackendKeyData"; + args = [reader.int32(), reader.int32()]; + break; + case 'Z': + type = "ReadyForQuery"; + args = [reader.string(1)]; + break; + case 'T': + type = "RowDescription"; + num_fields = reader.int16(); + data = []; + for (var i = 0; i < num_fields; i += 1) { + data.push({ + field: reader.cstring(), + table_id: reader.int32(), + column_id: reader.int16(), + type_id: reader.int32(), + type_size: reader.int16(), + type_modifier: reader.int32(), + format_code: reader.int16() + }); + } + args = [data]; + break; + case 'D': + type = "DataRow"; + data = []; + num_fields = reader.int16(); + for (i = 0; i < num_fields; i += 1) { + size = reader.int32(); + if (size === -1) { + data.push(null); + } else { + data.push(reader.string(size)); + } + } + args = [data]; + break; + case 'C': + type = "CommandComplete"; + args = [reader.cstring()]; + break; + case 'N': + type = "NoticeResponse"; + args = [{}]; + reader.multicstring().forEach(function (field) { + args[0][field[0]] = field.substr(1); + }); + break; + case '1': + type = 'ParseComplete'; + args = [{}]; + break; + case 'n': + type = 'NoData'; + args = []; + break; + case '2': + type = "BindComplete"; + args = [{}]; + break; + } + if (!type) { + sys.debug("Unknown response " + code); + } + return {type: type, args: args}; +} + +function message () { + var q = this; + var arr = []; + var pos = 0; + q.setMessages = function (msg) { + if (msg instanceof Array) { + arr = msg; } - args = [data]; - break; - case 'D': - type = "DataRow"; - data = []; - num_fields = reader.int16(); - for (i = 0; i < num_fields; i += 1) { - size = reader.int32(); - if (size === -1) { - data.push(null); - } else { - data.push(reader.string(size)); - } + } + q.next = function() { + if (arr[pos] !== null) { + pos = pos + 1; + return arr[pos-1]; } - args = [data]; - break; - case 'C': - type = "CommandComplete"; - args = [reader.cstring()]; - break; - case 'N': - type = "NoticeResponse"; - args = [{}]; - reader.multicstring().forEach(function (field) { - args[0][field[0]] = field.substr(1); - }); - break; - case '1': - type = 'ParseComplete'; - args = [{}]; - break; - case 'n': - type = 'NoData'; - args = []; - break; - case '2': - type = "BindComplete"; - args = [{}]; - break; - } - - - if (!type) { - sys.debug("Unknown response " + code); - } - return {type: type, args: args}; + return null; + } + q.addListener("RowDescription", function (desc) { + q.row_description = desc; + if (exports.DEBUG > 0) { + sys.debug("Caught RowDescription message."); + } + }); } +message.prototype = new process.EventEmitter; + function Query(sql, callback) { this.sql = sql; var q = this; - var row_description, results; + var results; results = []; /* Returns the next query object in this object buffer. This can be null. */ - var arr = [ + q.setMessages([ { type: 'Query', args: [sql] @@ -307,16 +330,7 @@ function Query(sql, callback) { type: 'Flush', args: [] } - ]; - var arrpos = 0; - q.next = function () { - arrpos += 1; - return arr[arrpos-1]; - }; - - q.addListener("RowDescription", function (desc) { - row_description = desc; - }); + ]); q.addListener("newRow", function (row) { results.push(row); }); @@ -344,7 +358,8 @@ function Prepared(sql, conn, callback) { var arr = [ { type: "Parse", - // Prepared name, the query, and a zero-length array to declare no types. + // Prepared name, the query, + // and a zero-length array to declare no types. args: [prepared_name, sql, []], }, { @@ -357,6 +372,8 @@ function Prepared(sql, conn, callback) { }, ]; // This describes a (nearly) complete lifecycle of a prepared statement. + q.setMessages(arr); + q.addListener("ParseComplete", function () { // Execute can now be run successfully. // Until this point, we can't assume that there's a matching query. @@ -369,11 +386,8 @@ function Prepared(sql, conn, callback) { q.emit("executable?"); }); - q.addListener("RowDescription", function (args) { - q.row_description = args; - if (exports.DEBUG > 0) { - sys.debug("Caught RowDescription in Prepared object."); - } + q.addListener("RowDescription", function (desc) { + // this should be called second. q.emit("executable?"); }); @@ -383,7 +397,7 @@ function Prepared(sql, conn, callback) { if (exports.DEBUG > 0) { sys.debug("hit executable?"); - sys.debug("parseComplete: "+parseComplete); + sys.debug("parseComplete: " + parseComplete); sys.debug("readyToExec: " + readyToExec); sys.debug((q.row_description != null && q.row_description.length > 0) || q.noData); } @@ -406,6 +420,7 @@ function Prepared(sql, conn, callback) { } }); + // Describes what parameters we should be sending. q.addListener("ParameterDescription", function (desc) { q.param_description = desc; q.emit("executable?"); @@ -424,29 +439,28 @@ function Prepared(sql, conn, callback) { q.execute = function () { // If the first argument is an array, then we use that as our bind // parameters. Otherwise, arguments[0] should be a function. + var args = Array.prototype.slice.call(arguments, 0); + var callback = null; + + if (typeof(args[args.length -1]) == 'function' ) { + callback = args.pop(); + } + // The rest of args is now the arguments to pass to the bound + // query, if any. + readyToExec = true; var eP; - if (arguments[0] instanceof Array) { - if (arguments[0].length >= 1) { - var callback = null; - // If the functions' a function, yay! - if (typeof(arguments[1]) == 'function') { - callback = arguments[1]; - } - eP = new execPrepared(prepared_name, q.param_description, arguments[0], q.row_description, conn, arguments[1]); - } - } - else if (typeof(arguments[0]) == 'function' ) { - eP = new execPrepared(prepared_name, q.param_description, [], q.row_description, conn, arguments[0]); - // Announcing that my slot is released in favour of this new query. - // The connection doesn't advance the buffer, but *does* replace - // the current query with this one. - } - else { - q.emit("error", "First argument must be array or function!"); - } + eP = new execPrepared( + prepared_name, + q.param_description, + args, + q.row_description, + conn, + callback + ); + if(exports.DEBUG > 0) { - sys.debug("Yielding"); + sys.debug("Yielding execute in prepare."); } conn.yield_to(q, eP); } @@ -459,13 +473,14 @@ function Prepared(sql, conn, callback) { return null; } } -Prepared.prototype = new process.EventEmitter; +Prepared.prototype = new message(); -function execPrepared (prepared, params, args, row_desc, conn, callback) { +function execPrepared (prepared, params, args, row_desc, callback) { var q = this; var portal = md5(prepared + args.join("|")); // Clearly need a generated portal name here. q.row_description = row_desc; var results = []; + var arr = [ { type: "Execute", @@ -476,6 +491,7 @@ function execPrepared (prepared, params, args, row_desc, conn, callback) { args: [portal, 0] } ]; + // If we have args, unshift the Bind. if (args instanceof Array && args.length >= 1) { arr.unshift({ @@ -484,6 +500,7 @@ function execPrepared (prepared, params, args, row_desc, conn, callback) { callback: callback }); } + q.setMessages(arr); q.addListener("BindComplete", function (args) { // We can mostly just ignore this, right? It's a notification of @@ -530,6 +547,16 @@ function execPrepared (prepared, params, args, row_desc, conn, callback) { execPrepared.prototype = new process.EventEmitter; +function Sync () { + var q = this; + q.setMessages([ + type: "Sync", + args: null + ]); +} + +Sync.prototype = new process.EventEmitter; + /* Initializes a connection to the database. DB connections are of the form: @@ -917,7 +944,104 @@ function Connection(args) { } }); + + conn.transaction = function (callback) { + var tx = new Transaction (conn); + tx.begin(); + callback(tx); // I don't like the immediate-mode of this. + } } Connection.prototype = new process.EventEmitter(); +/* Block object +A barebones object to represent multiple queries on the wire. +These are handed to the connectionManager to handle concurrency. To whit, +a "block" is a grouping of queries associated with some greater context. + +*/ +function Transaction (connection /*, params */) { + var conn = connection; // Associated connection. + var thisp = this; + var current_statement = null; + var messages = []; + // Error watcher, for wire errors. + + var message_length = 0; + + this.errors = new process.EventEmitter(); + + events = new process.EventEmitter(); + + events.on("queryAdded", function () { + message_length += 1; + }); + + events.on("queryCallback", function () { + + }); + + this.query = function (/* sql, some_args, callback */) { + var args = Array.prototype.slice(arguments); + var sql = args.shift(); + var callback = args.pop(); + if (args.length >0) { + // We now have a prepared query. + thisp.prepare(sql, function (sth) { + sth.execute(args, function (rs) { + callback(rs); + }); + }); + } + else { + // We have an otherwise normal query. + messages.push( + new Query(sql, callback); + ); + } + } + + this.prepare = function (sql, args, callback) { + + // Sets up a prepared query, and drops it onto the queue. + if (query.match(/\?/)) { + var i = 1; + query = query.replace(/\?/g, function () { return "$" + i++; }); + } + messages.push( + new Prepared(query, thisp, callback) + ); + events.emit("queryAdded"); + // conn.emit.call(conn, "queryAdded"); + } + + this._next_message = function () { + + } + + this.begin = function () { + // Begins the transaction. We now lock the transaction to the wire. + messages.push(thisp.query("BEGIN;")); // Don't need to watch for callbacks. + } + this.rollback = function () { + // Rolls back the request, and does the connection release. + + messages.push(thisp.query("ROLLBACK;")); + } + this.commit = function () { + // Commits this block of stuff that's happened, via the SYNC message. + // This will also cause a rollback if there's been errors. + messages.push( + new Sync(); + ); + } + + +} + +Context.prototype = new process.EventEmitter(); + +function connectionManager (dsn) { + // var conn = new Connection(dsn); +} + exports.connect = Connection; From 96ed8d376d41f67238ca6dadb48bd8e328321611 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Sun, 19 Sep 2010 14:38:56 -0700 Subject: [PATCH 26/54] Adding tranaction support into the core Connection object. Contemplating adding ReadyForQuery as a listener in the tx itself, so we can trap the messaging and next-query a little more effectively? --- lib/postgres-pure.js | 394 +++++++++++++++++++++++++++++-------------- 1 file changed, 269 insertions(+), 125 deletions(-) diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index a95a4d3..7994ba7 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -62,9 +62,10 @@ var formatter = { case "number": case "boolean": case "object": - builder.push.int16(1); // binary + builder.push.int16(1); // binary. This will need to be + // careful, as Date will be an object. break; - case "string": + case "string": // Any other type. builder.push.int16(0); break; } @@ -285,6 +286,7 @@ function parse_response(code, buffer) { return {type: type, args: args}; } + function message () { var q = this; var arr = []; @@ -339,7 +341,7 @@ function Query(sql, callback) { }); } -Query.prototype = new process.EventEmitter; +Query.prototype = new message; function Prepared(sql, conn, callback) { @@ -450,32 +452,22 @@ function Prepared(sql, conn, callback) { readyToExec = true; var eP; - eP = new execPrepared( + eP = new execute( prepared_name, q.param_description, - args, + args, // this could be blank. q.row_description, - conn, callback ); - if(exports.DEBUG > 0) { - sys.debug("Yielding execute in prepare."); + sys.debug("Pushing execute message to tx in Prepared."); } - conn.yield_to(q, eP); - } - - q.next = function() { - if (arr[pos] !== null) { - pos = pos + 1; - return arr[pos-1]; - } - return null; + conn.push(eP); } } Prepared.prototype = new message(); -function execPrepared (prepared, params, args, row_desc, callback) { +function execute (prepared, params, args, row_desc, callback) { var q = this; var portal = md5(prepared + args.join("|")); // Clearly need a generated portal name here. q.row_description = row_desc; @@ -527,35 +519,33 @@ function execPrepared (prepared, params, args, row_desc, callback) { if (exports.DEBUG > 0) { sys.debug("Calling with results"); } + // Callback gets wrapped at the tx layer, + // so we know when this gets tripped. callback.call(q, results); } else { this.type = data.type; + // Callback gets wrapped at the tx layer, + // so we know when this gets tripped. callback.call(q); } - conn.release(q); }); - var pos = 0; - q.next = function() { - if (arr[pos] !== null) { - pos = pos + 1; - return arr[pos-1]; - } - return null; - } } -execPrepared.prototype = new process.EventEmitter; +execute.prototype = new message; -function Sync () { +function Sync (callback) { var q = this; q.setMessages([ - type: "Sync", - args: null + { + type: "Sync", + args: null + }, ]); + // } -Sync.prototype = new process.EventEmitter; +Sync.prototype = new message; /* Initializes a connection to the database. @@ -573,7 +563,9 @@ function Connection(args) { args.username = args.auth[0]; args.password = args.auth[1]; } - var started, conn, connection, events, query_queue, current_query, results, readyState, closeState; + var started, conn, connection, + events, query_queue, current_query, + results, readyState, closeState; // Default to port 5432 args.port = args.port || 5432; @@ -613,6 +605,7 @@ function Connection(args) { } var queue = []; + /* Parses a message from the PG server */ function checkInput() { if (queue.length === 0) { return; } var first = queue[0]; @@ -633,7 +626,7 @@ function Connection(args) { return; } } - // What does this do? + // What does this do? -AS var message = first.slice(5, 5 + length); if (first.length === 5 + length) { queue.shift(); @@ -713,6 +706,7 @@ function Connection(args) { } }); + // Should this be handled at the tx level? events.addListener('ReadyForQuery', function () { if (exports.DEBUG > 0) { sys.debug("In RFQ"); @@ -722,17 +716,25 @@ function Connection(args) { conn.emit('connection'); } - if (closeState) { - connection.end(); - } else { - readyState = true; + + if (current_query instanceof process.EventEmitter) { + // Are there listeners for RFQ? + if (current_query.listeners("ReadyForQuery").length >= 1) { + + } } + else { + + + + } + if (exports.DEBUG > 0) { sys.debug(readyState); sys.debug("Queue length: "+query_queue.length); } - if (query_queue.length > 0 && readyState !== false) { + if (tx_queue.length > 0 && readyState !== false) { var query = query_queue.shift(); current_query = query; @@ -741,13 +743,19 @@ function Connection(args) { query_callback = query.callback; row_callback = query.row_callback; - // Implicit assumption we're only putting queries on the wire. + Implicit assumption we're only putting queries on the wire. sendMessage('Query', [query.sql]); */ readyState = false; events.emit("nextMessage"); - + } + else if (query_queue.length == 0) { + if (closeState) { + connection.end(); + } + // else + // do nothing, wait. } }); // This should always be caught by the current query. @@ -756,8 +764,10 @@ function Connection(args) { results = []; }); - // This is no longer correct. Sigh. + // Data row is handled by the connection for the time + // being, even though we should be looking at handling it in the + // query object, where the RowDescription lives. events.addListener("DataRow", function (data) { var row, i, l, description, value; row = {}; @@ -790,7 +800,7 @@ function Connection(args) { current_query.emit("newRow", row); } else { - results.push(row) + results.push(row); } }); @@ -808,92 +818,92 @@ function Connection(args) { // query_callback.call(this, results); }); - conn.query = function query(query) { - var parameters, callback; + conn.query = function query(/*query, args, callback */) { - // Grab the variable length parameters and the row_callback is there is one. - parameters = Array.prototype.slice.call(arguments, 1); + // Not sure I like this. + // I think this should be wrapped in a tx object. - if (typeof parameters[parameters.length - 1] === 'function') { - callback = parameters.pop(); - } - var q; - if (parameters.length == 1 && parameters[0] instanceof Array) { - // We have a parameterized query - q = new Prepared(query, function (sth) { - sth.execute(parameters[0], callback); + var tx = conn.tx(); + var args = Array.prototype.slice.call(arguments, 0); + + var callback = args.pop(); + + if (typeof(callback) === 'function') { + var cb = args.pop(); // remove the callback. + args.push(function (rs) { + // do the implicit sync/commit here? + // tx.commit(); + cb(rs, tx); // So they have access to the transaction? }); } else { - q = new Query(query, callback); - + // No callback. + args.push(function (rs) { + // This will eventually be handled by an autocommit flag in + // the parameters object as passed to the connection. + + tx.rollback(); // implicit rollback. + }); } - query_queue.push(q); + tx.query.apply(tx, args); events.emit("queryAdded"); }; conn.prepare = function prepare(query, callback) { // Sets up a prepared query, and drops it onto the queue. - if (query.match(/\?/)) { - var i = 1; - query = query.replace(/\?/g, function () { return "$" + i++; }); + + var tx = conn.tx(); // automatically pushes onto the stack. + var cb = null; + if (typeof(callback) === 'function') { + cb = function (sth) { + // do the implicit sync/commit here? + callback(sth, tx); + } + tx.prepared.call(tx, query, cb); + events.emit("queryAdded"); } - query_queue.push( - new Prepared(query, conn, callback) - ); - events.emit("queryAdded"); - // conn.emit.call(conn, "queryAdded"); + else { + // They didn't give us a prepared callback? That's.. odd. + conn.emit("error", "Cannot prepare query without callback"); + } + } + + // Sets up a new transaction object/block. + // Unsure if this should do an implicit BEGIN. + this.transaction = function () { + + var tx = new Transaction(this); + tx.on("ranCallback", function () { + if (exports.DEBUG >0 ) { + sys.debug("Received notification of callback execution."); + } + }); + conn.push(tx); + return tx; + } + // Alias function. + this.tx = function () { + return conn.transaction(); } - this.end = function () { + this.close = function () { closeState = true; // Close the connection right away if there are no pending queries - if (readyState) { + if (readyState) { connection.end(); } }; - events.addListener("queryAdded", function () { if (readyState) { conn.emit("ReadyForQuery"); } }); - /* Allows a currently-executing query to selectively modify the current - query queue, IF it is currently executing. - - Otherwise, it splices the query object in after its position in the queue. - */ - conn.yield_to = function (first, query) { - if (exports.DEBUG > 0) { - sys.debug("got yield_to"); - } - if (first === current_query) { - query_queue.unshift(query); - events.emit("queryAdded"); // Don't immediately switch to the next message. - } - else if (first in query_queue) { - // Splice it in after the query - query_queue.splice( query_queue.indexOf(first), 0, query ); - events.emit("queryAdded"); - } - } - - conn.release = function (query) { - if (exports.DEBUG > 0) { - sys.debug("got release"); - } - if (query === current_query) { - readyState = true; - events.emit("ReadyForQuery"); // Cycle along. - } - } // Pumps the current_query queue via .next() // and runs whatever we get back. events.addListener("nextMessage", function () { - if (events.DEBUG > 0) { sys.debug("got nextMessage"); } @@ -910,14 +920,36 @@ function Connection(args) { if (exports.DEBUG > 0) { sys.debug("Message is: "+msg); } - if (msg !== undefined && msg.type && msg.args) { // We have what we need to perform this query. sendMessage.apply(conn, [msg.type, msg.args]); } - } + } + }); + + events.addListener("nextQuery", function () { + if (events.DEBUG > 0) { + sys.debug("got nextQuery"); + } + if (tx_queue.length > 0) { + current_query = tx_query.pop(); + events.emit("nextMessage"); + } }); + // Pushes a new transaction into the transaction pool, assuming + // that the connection hasn't (yet) been closed. + // As transactions use internal buffers for query messages, this won't + // immediately interfere with attempts to add messages. + conn.push = function (tx) { + if (!closeState) { + tx_queue.push(tx); + } + else { + conn.emit("error", "Cannot add commands post-closure."); + } + } + conn.next = function (query) { if (query === current_query) { events.emit("nextMessage"); @@ -944,12 +976,6 @@ function Connection(args) { } }); - - conn.transaction = function (callback) { - var tx = new Transaction (conn); - tx.begin(); - callback(tx); // I don't like the immediate-mode of this. - } } Connection.prototype = new process.EventEmitter(); @@ -968,6 +994,11 @@ function Transaction (connection /*, params */) { var message_length = 0; + // Whether or not I can add more stuff to this transaction + // Marked true by the Connection. + var synced = false; + var closed = false; + this.errors = new process.EventEmitter(); events = new process.EventEmitter(); @@ -976,10 +1007,31 @@ function Transaction (connection /*, params */) { message_length += 1; }); - events.on("queryCallback", function () { - - }); + // This acts effectively as a decorator from Python + wrap = function (func) { + return function (rs) { + func(rs); + callbackRun(); + } + } + + // Marks that we've run a callback, usually + // after the callback is completed. + // This is so we can track the lifecycle of this + // transaction, and test whether or not the connction + // can release and move on to the next buffered transaction. + callbackRun = function () { + message_length -= 1; + } + // The basic Query declaration. + // This, by default, acts as a simple query when run without arguments + // (allowing for certain queries to be handled faster), + // and forcibly using the parameterized style in the event that arguments + // are passed, using PG's normal argument processing and + // escaping rules. + // This should act to reduce most instances of issue with people trying + // to write their own SQL escaping. this.query = function (/* sql, some_args, callback */) { var args = Array.prototype.slice(arguments); var sql = args.shift(); @@ -987,19 +1039,24 @@ function Transaction (connection /*, params */) { if (args.length >0) { // We now have a prepared query. thisp.prepare(sql, function (sth) { - sth.execute(args, function (rs) { - callback(rs); - }); + // Add the callback to the args list, so + args.push(wrap(callback)); + // we can use apply properly. + sth.execute.apply(sth, args); }); } else { // We have an otherwise normal query. - messages.push( - new Query(sql, callback); + // This does not require a normal Sync message + // or any other such magic-ery. + thisp.append( + new Query(sql, wrap(callback)); ); } } + // Standard prepared query. + this.prepare = function (sql, args, callback) { // Sets up a prepared query, and drops it onto the queue. @@ -1007,41 +1064,128 @@ function Transaction (connection /*, params */) { var i = 1; query = query.replace(/\?/g, function () { return "$" + i++; }); } - messages.push( - new Prepared(query, thisp, callback) + thisp.append( + new Prepared(query, thisp, wrap(callback)) ); events.emit("queryAdded"); // conn.emit.call(conn, "queryAdded"); } - this._next_message = function () { - + this.releaseable = function () { + // returns boolean + if (messages.length > 0 && synced == true) { + return false; + } + return true; + } + + this.close = function () { + closed = true; + thisp.sync(); } this.begin = function () { // Begins the transaction. We now lock the transaction to the wire. - messages.push(thisp.query("BEGIN;")); // Don't need to watch for callbacks. + thisp.append(thisp.query("BEGIN;")); // Don't need to watch for callbacks. } this.rollback = function () { // Rolls back the request, and does the connection release. - messages.push(thisp.query("ROLLBACK;")); + thisp.append(thisp.query("ROLLBACK;")); + thisp.sync(); } this.commit = function () { // Commits this block of stuff that's happened, via the SYNC message. // This will also cause a rollback if there's been errors. - messages.push( - new Sync(); - ); + thisp.append(thisp.query("COMMIT;")); + thisp.sync(); } + this.sync = function () { + thisp.append(new Sync( + function () { + callbackRun(); + } + )); + } + this.push = function (msg) { + if (!closed) { + messages.push(msg); + this.emit("queryAdded"); + } + else { + thisp.emit("error", "Transaction no longer valid!"); + } + } + this.next = function () { + if (messages.length > 0) { + return messages.shift(); // Front of the array, there. + } + return null; + } } Context.prototype = new process.EventEmitter(); -function connectionManager (dsn) { +// This will eventually be the return object +function connectionManager (dsn /*, connections=1 */) { // var conn = new Connection(dsn); } exports.connect = Connection; + + + +/* Allows a currently-executing query to selectively modify the current + query queue, IF it is currently executing. + + Otherwise, it splices the query object in after its position in the queue. +*/ +// conn.yield_to = function (first, query) { +// if (exports.DEBUG > 0) { +// sys.debug("got yield_to"); +// } +// if (first === current_query) { +// query_queue.unshift(query); +// events.emit("queryAdded"); // Don't immediately switch to the next message. +// } +// else if (first in query_queue) { +// // Splice it in after the query +// query_queue.splice( query_queue.indexOf(first), 0, query ); +// events.emit("queryAdded"); +// } +// } +// // Transactions shouldn't release themselves, I don't think. +// // +// conn.release = function (query) { +// if (exports.DEBUG > 0) { +// sys.debug("got release"); +// } +// if (query === current_query) { +// readyState = true; +// events.emit("ReadyForQuery"); // Cycle along. +// } +// } + + +// var parameters, callback; +// +// // Grab the variable length parameters and the row_callback is there is one. +// parameters = Array.prototype.slice.call(arguments, 1); +// +// if (typeof parameters[parameters.length - 1] === 'function') { +// callback = parameters.pop(); +// } +// var q; +// if (parameters.length == 1 && parameters[0] instanceof Array) { +// // We have a parameterized query +// q = new Prepared(query, function (sth) { +// sth.execute(parameters[0], callback); +// }); +// } +// else { +// q = new Query(query, callback); +// +// } +// query_queue.push(q); \ No newline at end of file From cc86ee65a975681edcde57e79ea02c21dcbe6f47 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Mon, 27 Sep 2010 20:28:19 -0700 Subject: [PATCH 27/54] Re-broke prepared queries. Mostly successfully migrated to the internal Transaction object metaphor. Queries are once again working successfully in the TX format. --- demo.js | 41 ++++++---- lib/postgres-pure.js | 181 +++++++++++++++++++++++++++++-------------- lib/results.js | 3 + 3 files changed, 150 insertions(+), 75 deletions(-) create mode 100644 lib/results.js diff --git a/demo.js b/demo.js index 8bdde6c..bd16d52 100644 --- a/demo.js +++ b/demo.js @@ -3,18 +3,29 @@ var pg = require("./lib/postgres-pure"); pg.DEBUG=0; var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); -db.prepare("SELECT ?::int", function (sth) { - sth.execute([1], function (rs) { - for (var i = 0; i < rs.length; i++) { - for (var key in rs[i]) { - if (rs[i].hasOwnProperty(key)) { - sys.puts(key +": " +rs[i][key]); - } - } - } - db.end(); - }); - sth.execute([2], function (rs) { - sys.puts(sys.inspect(rs)); - }) -}); \ No newline at end of file +db.query("SELECT 1::int as foobar;", function (rs) { + sys.puts(sys.inspect(rs)); + db.close(); +}); + +// db.prepare("SELECT ?::int AS foobar", function (sth) { +// sth.execute([1], function (rs) { +// sys.puts(sys.inspect(rs)); +// }); +// sth.execute([2], function (rs) { +// sys.puts(sys.inspect(rs)); +// +// }); +// }); +// +// db.transaction(function (tx) { +// tx.query("SELECT ?::int AS foobar", 1, function (rs) { +// sys.puts(sys.inspect(rs)); +// }); +// tx.prepare("SELECT ?::int AS foobar", function (sth) { +// sth.execute(2, function (rs) { +// sys.puts(sys.inspect(rs)); +// }); +// }); +// tx.commit(); +// }); \ No newline at end of file diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 7994ba7..3841b84 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -344,8 +344,10 @@ function Query(sql, callback) { Query.prototype = new message; function Prepared(sql, conn, callback) { + // Per #postgresql on freenode, use the unnamed prepared statement to + // increase performance for queries. - var prepared_name = md5(sql); // Use the md5 hash. This is easily selectable later. + // var prepared_name = md5(sql); // Use the md5 hash. This is easily selectable later. var q = this; q.row_description = null; q.param_description = null; @@ -362,11 +364,11 @@ function Prepared(sql, conn, callback) { type: "Parse", // Prepared name, the query, // and a zero-length array to declare no types. - args: [prepared_name, sql, []], + args: ['', sql, []], }, { type: "Describe", - args: ["S", prepared_name], + args: ["S", ''], // The unnamed prepare. }, { type: "Flush", @@ -565,7 +567,8 @@ function Connection(args) { } var started, conn, connection, events, query_queue, current_query, - results, readyState, closeState; + results, readyState, closeState, + tx_queue, current_tx; // Default to port 5432 args.port = args.port || 5432; @@ -575,12 +578,14 @@ function Connection(args) { connection = net.createConnection(args.port, args.hostname); events = new process.EventEmitter(); - query_queue = []; readyState = false; closeState = false; started = false; conn = this; current_query = null; + current_tx = null + tx_queue = []; + results = []; // Disable the idle timeout on the connection connection.setTimeout(0); @@ -663,7 +668,7 @@ function Connection(args) { } else { if (exports.DEBUG > 0) { - sys.debug("Sending "+command.type+" to local handler"); + sys.debug("Sending "+command.type+" to local handler"); } events.emit.apply(events, command.args); } @@ -707,7 +712,7 @@ function Connection(args) { }); // Should this be handled at the tx level? - events.addListener('ReadyForQuery', function () { + events.on('ReadyForQuery', function (state) { if (exports.DEBUG > 0) { sys.debug("In RFQ"); } @@ -716,48 +721,70 @@ function Connection(args) { conn.emit('connection'); } - - if (current_query instanceof process.EventEmitter) { - // Are there listeners for RFQ? - if (current_query.listeners("ReadyForQuery").length >= 1) { - - } - } - else { - - - - } + // We can cycle to the next? if (exports.DEBUG > 0) { - sys.debug(readyState); - sys.debug("Queue length: "+query_queue.length); + sys.debug("Readystate is: " + readyState); + sys.debug("Queue length: "+tx_queue.length); } - if (tx_queue.length > 0 && readyState !== false) { + if (readyState) { + // assumed to be set by nextMessage, + // stating that we're out of messages in this particular + // transaction. + // Ergo, cycle to the next transaction and call nextMessage. - var query = query_queue.shift(); - current_query = query; + if (current_tx !== null) { + if (current_tx.can_release()) { + // We now push the implicit + // commit/sync messages onto the stack, if necessary. + if (state === "T" || state === "E") { + // Sync/rollback explicitly. + current_tx.rollback(); + } + } + } - /* - query_callback = query.callback; - row_callback = query.row_callback; + if (tx_queue.length > 0 && readyState) { + + current_tx = tx_queue.shift(); + + /* + query_callback = query.callback; + row_callback = query.row_callback; + + Implicit assumption we're only putting queries on the wire. + sendMessage('Query', [query.sql]); + */ + + readyState = false; + events.emit("nextMessage"); + } - Implicit assumption we're only putting queries on the wire. - sendMessage('Query', [query.sql]); - */ - readyState = false; + else if (tx_queue.length === 0) { + if (closeState) { + connection.end(); + } + } + } + else if (current_query === null) { + if (exports.DEBUG > 0) { + sys.debug("Current query is null, emitting nextMessage"); + } events.emit("nextMessage"); } - else if (query_queue.length == 0) { + else if (tx_queue.length === 0) { if (closeState) { connection.end(); } - // else - // do nothing, wait. } + // else { + // // We should trigger the nextMessage, since we're ready. + // } }); + + // This should always be caught by the current query. events.addListener("RowDescription", function (data) { row_description = data; @@ -776,9 +803,9 @@ function Connection(args) { description = current_query.row_description[i]; value = data[i]; if (value !== null) { - // TODO: investigate to see if these numbers are stable across databases or - // if we need to dynamically pull them from the pg_types table + // Type OIDs are stable. // TODO: Investigate javascript Date objects. + // see pg_type.h for the defined values. switch (description.type_id) { case 16: // bool value = value === 't'; @@ -788,6 +815,7 @@ function Connection(args) { case 23: // int4 value = parseInt(value, 10); break; + } } row[description.field] = value; @@ -824,25 +852,25 @@ function Connection(args) { // I think this should be wrapped in a tx object. var tx = conn.tx(); - var args = Array.prototype.slice.call(arguments, 0); + var args = Array.prototype.slice.call(arguments); var callback = args.pop(); if (typeof(callback) === 'function') { - var cb = args.pop(); // remove the callback. args.push(function (rs) { // do the implicit sync/commit here? // tx.commit(); - cb(rs, tx); // So they have access to the transaction? + callback(rs, tx); // So they have access to the transaction? }); } else { // No callback. + args.push(callback); // re-add it. args.push(function (rs) { // This will eventually be handled by an autocommit flag in // the parameters object as passed to the connection. - tx.rollback(); // implicit rollback. + //tx.(); // implicit rollback. }); } tx.query.apply(tx, args); @@ -859,7 +887,10 @@ function Connection(args) { // do the implicit sync/commit here? callback(sth, tx); } - tx.prepared.call(tx, query, cb); + if (exports.DEBUG > 0) { + sys.debug("Transaction is: " + tx); + } + tx.prepare.call(tx, query, cb); events.emit("queryAdded"); } else { @@ -870,6 +901,7 @@ function Connection(args) { // Sets up a new transaction object/block. // Unsure if this should do an implicit BEGIN. + // Maybe a setting? this.transaction = function () { var tx = new Transaction(this); @@ -894,7 +926,7 @@ function Connection(args) { connection.end(); } }; - events.addListener("queryAdded", function () { + events.on("queryAdded", function () { if (readyState) { conn.emit("ReadyForQuery"); } @@ -903,27 +935,51 @@ function Connection(args) { // Pumps the current_query queue via .next() // and runs whatever we get back. - events.addListener("nextMessage", function () { + events.on("nextMessage", function () { if (events.DEBUG > 0) { sys.debug("got nextMessage"); } var msg; + + if (current_query === null && current_tx !== null) { + current_query = current_tx.next(); // Get the next off the buffer. + } + if (current_query !== null) { if (exports.DEBUG > 0) { - sys.debug("current query is not Null."); + sys.debug("nextMessage: current query is not Null."); } msg = current_query.next(); if (exports.DEBUG > 0) { sys.debug("Message is: "+msg); + sys.debug("Message typeof is: " + typeof(msg)); } - if (msg !== undefined && msg.type && msg.args) { - // We have what we need to perform this query. - sendMessage.apply(conn, [msg.type, msg.args]); + if (msg !== undefined) { + if (msg !== null && msg.type && msg.args) { + // We have what we need to perform this query. + if (exports.DEBUG > 0) { + sys.debug("Sending message: " + msg.type + ": " + msg.args); + } + sendMessage.apply(conn, [msg.type, msg.args]); + } } + // else { + // // If it's null, it's out of messages. + // // Get the next msg block from the tx. + // current_query = current_tx.next(); + // events.emit("nextMessage"); + // } + } + else { + // current_tx was null, or empty. + // Since we appear to be ready to handle a new query, + // we begin handling a new query. + readyState = true; + events.emit("ReadyForQuery", "T"); // In a transaction, as far as we know. } }); @@ -1032,8 +1088,13 @@ function Transaction (connection /*, params */) { // escaping rules. // This should act to reduce most instances of issue with people trying // to write their own SQL escaping. - this.query = function (/* sql, some_args, callback */) { - var args = Array.prototype.slice(arguments); + + /* sql, some_args, callback */ + this.query = function () { + var args = Array.prototype.slice.call(arguments); + if (exports.DEBUG>0) { + sys.debug("Args are: " + args); + } var sql = args.shift(); var callback = args.pop(); if (args.length >0) { @@ -1049,8 +1110,8 @@ function Transaction (connection /*, params */) { // We have an otherwise normal query. // This does not require a normal Sync message // or any other such magic-ery. - thisp.append( - new Query(sql, wrap(callback)); + thisp.push( + new Query(sql, wrap(callback)) ); } } @@ -1060,18 +1121,18 @@ function Transaction (connection /*, params */) { this.prepare = function (sql, args, callback) { // Sets up a prepared query, and drops it onto the queue. - if (query.match(/\?/)) { + if (sql.match(/\?/)) { var i = 1; - query = query.replace(/\?/g, function () { return "$" + i++; }); + sql = sql.replace(/\?/g, function () { return "$" + i++; }); } - thisp.append( - new Prepared(query, thisp, wrap(callback)) + thisp.push( + new Prepared(sql, thisp, wrap(callback)) ); events.emit("queryAdded"); // conn.emit.call(conn, "queryAdded"); } - this.releaseable = function () { + this.can_release = function () { // returns boolean if (messages.length > 0 && synced == true) { return false; @@ -1091,18 +1152,18 @@ function Transaction (connection /*, params */) { this.rollback = function () { // Rolls back the request, and does the connection release. - thisp.append(thisp.query("ROLLBACK;")); + thisp.push(thisp.query("ROLLBACK;")); thisp.sync(); } this.commit = function () { // Commits this block of stuff that's happened, via the SYNC message. // This will also cause a rollback if there's been errors. - thisp.append(thisp.query("COMMIT;")); + thisp.push(thisp.query("COMMIT;")); thisp.sync(); } this.sync = function () { - thisp.append(new Sync( + thisp.push(new Sync( function () { callbackRun(); } @@ -1126,7 +1187,7 @@ function Transaction (connection /*, params */) { } } -Context.prototype = new process.EventEmitter(); +Transaction.prototype = new process.EventEmitter(); // This will eventually be the return object function connectionManager (dsn /*, connections=1 */) { diff --git a/lib/results.js b/lib/results.js new file mode 100644 index 0000000..97e28de --- /dev/null +++ b/lib/results.js @@ -0,0 +1,3 @@ +function result (desc, row) { + this.description = desc; +} \ No newline at end of file From 54b003535e2dc917dbe5931d31f64f3ed12de6d3 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Thu, 30 Sep 2010 08:03:45 -0700 Subject: [PATCH 28/54] Multiple queries & transactioned queries now working correctly. Prepared queries are still broken. Underlying pipeline/query buffer mechanism appears to be working correctly. --- demo.js | 18 ++- lib/postgres-pure.js | 373 ++++++++++++++++++++++--------------------- 2 files changed, 200 insertions(+), 191 deletions(-) diff --git a/demo.js b/demo.js index bd16d52..26af57e 100644 --- a/demo.js +++ b/demo.js @@ -1,22 +1,28 @@ var sys = require("sys"); var pg = require("./lib/postgres-pure"); -pg.DEBUG=0; +pg.DEBUG=1; var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); -db.query("SELECT 1::int as foobar;", function (rs) { +db.query("SELECT 1::int as foobar;", function (rs, tx) { sys.puts(sys.inspect(rs)); - db.close(); + tx.query("SELECT 2::int as foobartwo", function (rs) { + sys.puts(sys.inspect(rs)); + }); + // db.close(); }); -// db.prepare("SELECT ?::int AS foobar", function (sth) { -// sth.execute([1], function (rs) { +// db.prepare("SELECT ?::int AS foobar", function (sth, tx) { +// sth.execute(1, function (rs) { // sys.puts(sys.inspect(rs)); // }); -// sth.execute([2], function (rs) { +// sth.execute(2, function (rs) { // sys.puts(sys.inspect(rs)); // // }); +// // db.close(); // }); +db.close(); + // // db.transaction(function (tx) { // tx.query("SELECT ?::int AS foobar", 1, function (rs) { diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 3841b84..d01f353 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -290,35 +290,40 @@ function parse_response(code, buffer) { function message () { var q = this; var arr = []; - var pos = 0; + var pos = 0; // instance variable q.setMessages = function (msg) { if (msg instanceof Array) { arr = msg; } + q.length = arr.length; } q.next = function() { - if (arr[pos] !== null) { + sys.debug("pos: " + pos); + if (arr[pos] !== null && arr[pos] !== undefined) { pos = pos + 1; return arr[pos-1]; } + sys.debug("pos " + pos + " exceeds internal message buffer."); + sys.debug(sys.inspect(arr)); return null; } - q.addListener("RowDescription", function (desc) { - q.row_description = desc; - if (exports.DEBUG > 0) { - sys.debug("Caught RowDescription message."); + q.empty = function () { + if (pos >= arr.length) { + return true; } - }); + return false; + } } message.prototype = new process.EventEmitter; function Query(sql, callback) { + message.call(this); this.sql = sql; var q = this; - var results; - results = []; + q.results = []; + var pos = 0; /* Returns the next query object in this object buffer. This can be null. @@ -328,30 +333,41 @@ function Query(sql, callback) { type: 'Query', args: [sql] }, - { - type: 'Flush', - args: [] - } + // { + // type: 'Flush', + // args: [] + // } ]); q.addListener("newRow", function (row) { - results.push(row); + q.results.push(row); }); q.addListener("Complete", function (data) { - callback(results); + sys.debug("Callback: " + callback); + callback(q.results); + }); + q.toString = function () { return "Query: " + q.length}; + q.addListener("RowDescription", function (desc) { + q.row_description = desc; + if (exports.DEBUG > 0) { + sys.debug("Caught RowDescription message."); + } }); } -Query.prototype = new message; +Query.prototype = new message(); +Query.prototype.constructor = Query; function Prepared(sql, conn, callback) { // Per #postgresql on freenode, use the unnamed prepared statement to // increase performance for queries. // var prepared_name = md5(sql); // Use the md5 hash. This is easily selectable later. - var q = this; - q.row_description = null; - q.param_description = null; - q.noData = false; + + var self = this; + message.call(this); + self.row_description = null; + self.param_description = null; + self.noData = false; var parseComplete = null; var readyToExec = false; @@ -376,9 +392,9 @@ function Prepared(sql, conn, callback) { }, ]; // This describes a (nearly) complete lifecycle of a prepared statement. - q.setMessages(arr); + self.setMessages(arr); - q.addListener("ParseComplete", function () { + self.addListener("ParseComplete", function () { // Execute can now be run successfully. // Until this point, we can't assume that there's a matching query. // Anyway, we now run a DESCRIBE operation, and store the row @@ -387,51 +403,22 @@ function Prepared(sql, conn, callback) { // connection object. // conn.next(); parseComplete = true; - q.emit("executable?"); }); - q.addListener("RowDescription", function (desc) { + self.on("RowDescription", function (desc) { // this should be called second. - q.emit("executable?"); + self.row_description = desc; }); var execute = []; - q.addListener("executable?", function () { - - if (exports.DEBUG > 0) { - sys.debug("hit executable?"); - sys.debug("parseComplete: " + parseComplete); - sys.debug("readyToExec: " + readyToExec); - sys.debug((q.row_description != null && q.row_description.length > 0) || q.noData); - } - - if (parseComplete && readyToExec && ((q.row_description != null && q.row_description.length > 0) || q.noData)) { - conn.release(q); // Yield myself. - if (exports.DEBUG > 0) { - sys.debug("Yielding to the next message set."); - } - } - else if (parseComplete && ((q.row_description != null && q.row_description.length > 0) || q.noData)) { - // The server is prepped. We can now safely run the callback - // and trap the execute statements, prior to adding them to the - // main query queue. - if (exports.DEBUG > 0) { - sys.debug("calling callback"); - } - callback(q); - q.emit("executable?"); - } - }); - // Describes what parameters we should be sending. - q.addListener("ParameterDescription", function (desc) { - q.param_description = desc; - q.emit("executable?"); + self.on("ParameterDescription", function (desc) { + self.param_description = desc; }); - q.addListener("NoData", function () { - q.noData = true; + self.on("NoData", function () { + self.noData = true; }) /* @@ -440,7 +427,7 @@ function Prepared(sql, conn, callback) { If we have been, then we need to issue a BIND on the wire. If we haven't been, we can move straight to EXECUTE. */ - q.execute = function () { + self.execute = function () { // If the first argument is an array, then we use that as our bind // parameters. Otherwise, arguments[0] should be a function. var args = Array.prototype.slice.call(arguments, 0); @@ -456,9 +443,9 @@ function Prepared(sql, conn, callback) { var eP; eP = new execute( prepared_name, - q.param_description, + self.param_description, args, // this could be blank. - q.row_description, + self.row_description, callback ); if(exports.DEBUG > 0) { @@ -468,6 +455,7 @@ function Prepared(sql, conn, callback) { } } Prepared.prototype = new message(); +Prepared.prototype.constructor = Prepared; function execute (prepared, params, args, row_desc, callback) { var q = this; @@ -483,6 +471,10 @@ function execute (prepared, params, args, row_desc, callback) { { type: "Flush", args: [portal, 0] + }, + { + type: "Sync", + args: [] } ]; @@ -568,7 +560,7 @@ function Connection(args) { var started, conn, connection, events, query_queue, current_query, results, readyState, closeState, - tx_queue, current_tx; + tx_queue, current_tx, queryEmpty; // Default to port 5432 args.port = args.port || 5432; @@ -586,6 +578,8 @@ function Connection(args) { current_tx = null tx_queue = []; results = []; + queryEmpty = true; + var wait = false; // Disable the idle timeout on the connection connection.setTimeout(0); @@ -603,10 +597,11 @@ function Connection(args) { } } connection.write(buffer); - if (current_query) { - conn.next(current_query); // We don't always expect to get a response message. - // And if we do, the message object can sort it out. - } + events.emit("nextMessage"); + // if (current_query) { + // conn.next(current_query); // We don't always expect to get a response message. + // // And if we do, the message object can sort it out. + // } } var queue = []; @@ -725,63 +720,118 @@ function Connection(args) { if (exports.DEBUG > 0) { sys.debug("Readystate is: " + readyState); - sys.debug("Queue length: "+tx_queue.length); + sys.debug("TX Queue length: "+tx_queue.length); + sys.debug("TX is "+ current_tx); + } + readyState = true; + if (current_tx === null ) { + events.emit("nextTx"); + } + else if (current_query === null || current_query === undefined || current_query.empty()) { + events.emit("nextQuery"); + } + else { + // See if we can move forwards. + events.emit("nextMessage"); + } + }); + + events.on("canClose?", function () { + if (tx_queue.length == 0 && + current_tx.can_release() && + (current_query === null || + current_query.empty()) && + closeState && readyState + ) { + connection.end(); + } + }); + + // Pumps the current_query queue via .next() + // and runs whatever we get back. + events.on("nextMessage", function () { + if (exports.DEBUG > 0) { + sys.debug("got nextMessage"); } - if (readyState) { - // assumed to be set by nextMessage, - // stating that we're out of messages in this particular - // transaction. - // Ergo, cycle to the next transaction and call nextMessage. - - if (current_tx !== null) { - if (current_tx.can_release()) { - // We now push the implicit - // commit/sync messages onto the stack, if necessary. - if (state === "T" || state === "E") { - // Sync/rollback explicitly. - current_tx.rollback(); - } + // pulls the next message from the current query. + // If it's null, it calls nextQuery. + + if (current_query === null) { + sys.debug("nextMessage: Getting next message block"); + events.emit("nextQuery"); + } + else if (!current_query.empty()) { + var msg = current_query.next(); + sys.debug(msg); + if (msg && msg.type && msg.args) { + // We have what we need to perform this query. + if (exports.DEBUG > 0) { + sys.debug("Sending message: " + msg.type + ": " + msg.args); } - } - - if (tx_queue.length > 0 && readyState) { - - current_tx = tx_queue.shift(); - - /* - query_callback = query.callback; - row_callback = query.row_callback; - - Implicit assumption we're only putting queries on the wire. - sendMessage('Query', [query.sql]); - */ - readyState = false; - events.emit("nextMessage"); + sendMessage.apply(conn, [msg.type, msg.args]); } - - - else if (tx_queue.length === 0) { - if (closeState) { - connection.end(); - } + else { + sys.debug("getting next query, nM"); + events.emit("nextQuery"); } } - else if (current_query === null) { - if (exports.DEBUG > 0) { - sys.debug("Current query is null, emitting nextMessage"); + else { + // wait + sys.debug("waiting for query buffer"); + } + }); + + events.addListener("nextQuery", function () { + if (exports.DEBUG > 0) { + sys.debug("got nextQuery"); + } + // if (readyState) { + if ( current_tx !== null && current_tx !== undefined) { + current_query = current_tx.next(); + if (current_query !== null && current_query !== undefined) { + events.emit("nextMessage"); + } + else { + if (exports.DEBUG > 0) { + sys.debug("next query is null."); + } + events.emit("nextTx"); + } } - events.emit("nextMessage"); + else { + if (exports.DEBUG > 0) { + sys.debug("calling canClose? from nextQuery"); + } + events.emit("canClose?"); + } + // } + }); + + events.on("nextTx", function () { + if (exports.DEBUG > 0) { + sys.debug("got nextTx"); } - else if (tx_queue.length === 0) { - if (closeState) { - connection.end(); + if (readyState) { + if (tx_queue.length > 0) { + current_tx = tx_queue.shift(); + if (current_tx !== null && current_tx !== undefined) { + events.emit("nextQuery"); + } + else { + if (exports.DEBUG > 0) { + sys.debug("next TX is null."); + } + } + } + else { + if (exports.DEBUG > 0) { + sys.debug("calling canClose? from nextTx"); + } + events.emit("canClose?"); } } - // else { - // // We should trigger the nextMessage, since we're ready. - // } }); @@ -844,13 +894,13 @@ function Connection(args) { current_query.emit("Complete", data); } // query_callback.call(this, results); + //readyState = true; }); conn.query = function query(/*query, args, callback */) { // Not sure I like this. // I think this should be wrapped in a tx object. - var tx = conn.tx(); var args = Array.prototype.slice.call(arguments); @@ -922,76 +972,16 @@ function Connection(args) { closeState = true; // Close the connection right away if there are no pending queries - if (readyState) { + if (readyState) { connection.end(); } }; - events.on("queryAdded", function () { - if (readyState) { - conn.emit("ReadyForQuery"); - } - }); - - - // Pumps the current_query queue via .next() - // and runs whatever we get back. - events.on("nextMessage", function () { - if (events.DEBUG > 0) { - sys.debug("got nextMessage"); - } - - var msg; - - if (current_query === null && current_tx !== null) { - current_query = current_tx.next(); // Get the next off the buffer. - } - - if (current_query !== null) { - - if (exports.DEBUG > 0) { - sys.debug("nextMessage: current query is not Null."); - } - - msg = current_query.next(); - - if (exports.DEBUG > 0) { - sys.debug("Message is: "+msg); - sys.debug("Message typeof is: " + typeof(msg)); - } - if (msg !== undefined) { - if (msg !== null && msg.type && msg.args) { - // We have what we need to perform this query. - if (exports.DEBUG > 0) { - sys.debug("Sending message: " + msg.type + ": " + msg.args); - } - sendMessage.apply(conn, [msg.type, msg.args]); - } - } - // else { - // // If it's null, it's out of messages. - // // Get the next msg block from the tx. - // current_query = current_tx.next(); - // events.emit("nextMessage"); - // } - } - else { - // current_tx was null, or empty. - // Since we appear to be ready to handle a new query, - // we begin handling a new query. - readyState = true; - events.emit("ReadyForQuery", "T"); // In a transaction, as far as we know. - } - }); + // events.on("queryAdded", function () { + // if (readyState) { + // conn.emit("ReadyForQuery"); + // } + // }); - events.addListener("nextQuery", function () { - if (events.DEBUG > 0) { - sys.debug("got nextQuery"); - } - if (tx_queue.length > 0) { - current_query = tx_query.pop(); - events.emit("nextMessage"); - } - }); // Pushes a new transaction into the transaction pool, assuming // that the connection hasn't (yet) been closed. @@ -1065,10 +1055,11 @@ function Transaction (connection /*, params */) { // This acts effectively as a decorator from Python wrap = function (func) { - return function (rs) { + return (function (rs) { + func(rs); callbackRun(); - } + }); } // Marks that we've run a callback, usually @@ -1110,9 +1101,15 @@ function Transaction (connection /*, params */) { // We have an otherwise normal query. // This does not require a normal Sync message // or any other such magic-ery. - thisp.push( - new Query(sql, wrap(callback)) - ); + var q = new Query(sql, function (rs) { + sys.debug("RS is:" + sys.inspect(rs)); + callback(rs); + callbackRun(); + }); + if (exports.DEBUG > 0) { + sys.debug("New plain query created: "+q); + } + thisp.push(q); } } @@ -1134,7 +1131,7 @@ function Transaction (connection /*, params */) { this.can_release = function () { // returns boolean - if (messages.length > 0 && synced == true) { + if (messages.length > 0) { return false; } return true; @@ -1174,6 +1171,9 @@ function Transaction (connection /*, params */) { if (!closed) { messages.push(msg); this.emit("queryAdded"); + if (exports.DEBUG > 0) { + sys.debug("Added message of " + msg + " to TX"); + } } else { thisp.emit("error", "Transaction no longer valid!"); @@ -1181,8 +1181,11 @@ function Transaction (connection /*, params */) { } this.next = function () { if (messages.length > 0) { - return messages.shift(); // Front of the array, there. + if (messages[0] !== null && messages[0] !== undefined) { + return messages.shift(); // Front of the array, there. + } } + sys.debug("Returning null from tx.next()"); return null; } } From f56cee15867bd74348d700e12c9df76972bf97c7 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Sat, 9 Oct 2010 21:16:05 -0700 Subject: [PATCH 29/54] Prepared queries work As Expected. Lots of debugging symbols still defaulting to on, that need to be cleaned up. Other code that needs to be written: * Disengagement code, so that a messages block that's retired can be primed when it is placed back onto the query buffer. (say a .execute gets run later.) * LISTEN. It makes so much sense. --- demo.js | 22 +-- lib/postgres-pure.js | 386 ++++++++++++++++++++++++++++--------------- 2 files changed, 260 insertions(+), 148 deletions(-) diff --git a/demo.js b/demo.js index 26af57e..fc7f8a3 100644 --- a/demo.js +++ b/demo.js @@ -1,6 +1,6 @@ var sys = require("sys"); var pg = require("./lib/postgres-pure"); -pg.DEBUG=1; +// pg.DEBUG=1; var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); db.query("SELECT 1::int as foobar;", function (rs, tx) { @@ -11,16 +11,16 @@ db.query("SELECT 1::int as foobar;", function (rs, tx) { // db.close(); }); -// db.prepare("SELECT ?::int AS foobar", function (sth, tx) { -// sth.execute(1, function (rs) { -// sys.puts(sys.inspect(rs)); -// }); -// sth.execute(2, function (rs) { -// sys.puts(sys.inspect(rs)); -// -// }); -// // db.close(); -// }); +db.prepare("SELECT ?::int AS foobar", function (sth, tx) { + sth.execute(1, function (rs, tx) { + sys.puts(sys.inspect(rs)); + }); + sth.execute(2, function (rs) { + sys.puts(sys.inspect(rs)); + + }); + // db.close(); +}); db.close(); // diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index d01f353..a6518c5 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -288,27 +288,36 @@ function parse_response(code, buffer) { function message () { - var q = this; - var arr = []; + var self = this; + this.messages = []; var pos = 0; // instance variable - q.setMessages = function (msg) { + this.setMessages = function (msg) { if (msg instanceof Array) { - arr = msg; + self.messages = msg; + } + self.length = self.messages.length; + } + this.addMessage = function (msg) { + if (msg != null && msg != undefined) { + self.messages.push(msg); + self.length = self.messages.length; } - q.length = arr.length; } - q.next = function() { + this.next = function() { sys.debug("pos: " + pos); - if (arr[pos] !== null && arr[pos] !== undefined) { + sys.debug("messages:" + sys.inspect(self.messages)); + if (self.messages[pos] !== null && self.messages[pos] !== undefined) { pos = pos + 1; - return arr[pos-1]; + sys.debug("Returning: " + self.messages[pos-1]); + return self.messages[pos-1]; } sys.debug("pos " + pos + " exceeds internal message buffer."); - sys.debug(sys.inspect(arr)); + sys.debug("Returning null"); return null; } - q.empty = function () { - if (pos >= arr.length) { + this.empty = function () { + sys.debug("message pos " + pos + " exceeds length "+ self.messages.length +": " + (pos >= self.messages.length)); + if (pos >= self.messages.length) { return true; } return false; @@ -357,22 +366,22 @@ function Query(sql, callback) { Query.prototype = new message(); Query.prototype.constructor = Query; -function Prepared(sql, conn, callback) { +function Prepared(sql, conn /*, use_named */) { // Per #postgresql on freenode, use the unnamed prepared statement to // increase performance for queries. // var prepared_name = md5(sql); // Use the md5 hash. This is easily selectable later. var self = this; - message.call(this); + // message.call(this); self.row_description = null; - self.param_description = null; + self.parameters = null; self.noData = false; var parseComplete = null; var readyToExec = false; var conn = conn; - + var name = ''; var pos = 0; var callback = callback; var arr = [ @@ -386,10 +395,10 @@ function Prepared(sql, conn, callback) { type: "Describe", args: ["S", ''], // The unnamed prepare. }, - { - type: "Flush", - args: [], - }, + // { + // type: "Flush", + // args: [], + // }, ]; // This describes a (nearly) complete lifecycle of a prepared statement. self.setMessages(arr); @@ -414,12 +423,15 @@ function Prepared(sql, conn, callback) { // Describes what parameters we should be sending. self.on("ParameterDescription", function (desc) { - self.param_description = desc; + self.parameters = desc; }); self.on("NoData", function () { self.noData = true; }) + self.on("newRow", function (row) { + currExec.emit("newRow", row); + }); /* Executing the function tests whether or not @@ -427,106 +439,218 @@ function Prepared(sql, conn, callback) { If we have been, then we need to issue a BIND on the wire. If we haven't been, we can move straight to EXECUTE. */ + var eB = []; + var currExec = null; + var cPos = 0; + self.on("Complete", function (data) { + if (currExec != null) { + currExec.emit("Complete", data); // Fires the callback. + currExec = eB[cPos++]; // So we don't lose the reference to the + // execute command. + if (currExec != null && currExec != undefined) { + currExec.args.forEach(function (i) { + sys.debug("Arg is: " + i.type); + self.addMessage(i); + }); + } + } + }); + + self.on("BindComplete", function () { + if (currExec != null) { + currExec.emit("BindComplete"); + } + }); + + // self.empty = function () { + // return self.__proto__.empty.call(self); + // } + + self.next = function () { + // Override. + var msg = this.__proto__.next(); + // if ( exports.DEBUG > 0) { + // sys.debug("Message is: " + msg); + // sys.debug("proto is: " + self.__proto__); + // } + sys.debug("currexec " + currExec); + sys.debug("msg" + msg); + if (msg !== null) { + return msg; + } + else if (currExec === null) { + + if (exports.DEBUG > 0) { + sys.debug("eB is " + eB); + } + if (eB[cPos] !== null && eB[cPos] !== undefined) { + currExec = eB[cPos++]; // So we don't lose the reference to the + // execute command. + sys.debug(sys.inspect(currExec.args.slice(1))); + currExec.args.forEach(function (i) { + sys.debug("Arg is: " + i.type); + self.addMessage(i); + }); + // for (arg in currExec.args.slice(1)) { + // + // if (currExec.args.hasOwnProperty(arg)) { + // + // } + // } + // self.addMessage({ + // type: "Flush", + // args: [], + // }); // Flush the current query out. + sys.debug("Calling next from prototype."); + return self.__proto__.next(); + } + } + sys.debug("currExec isn't null, returning null from next"); + return null; // Patience.. + } + self.execute = function () { // If the first argument is an array, then we use that as our bind // parameters. Otherwise, arguments[0] should be a function. + if (exports.DEBUG > 0) { + sys.debug("Execute got called."); + } var args = Array.prototype.slice.call(arguments, 0); var callback = null; if (typeof(args[args.length -1]) == 'function' ) { callback = args.pop(); } + else { + // No callback? That's an error. + self.emit("error", "Cannot execute without callback."); + return; + } // The rest of args is now the arguments to pass to the bound // query, if any. readyToExec = true; var eP; - eP = new execute( - prepared_name, - self.param_description, - args, // this could be blank. - self.row_description, - callback - ); + eP = new process.EventEmitter(); + //eP.portal = md5(name + args.join("|")); // Clearly need a generated portal name here. + eP.portal = ''; + eP.bound = false; + eP.results = []; + eP.args = [ + { + type: "Execute", + args: [eP.portal, 0], + }, + { + type: "Flush", + args: [] + }, + { + type: "Sync", + args: [] + } + ]; + if (args.length > 0) { + eP.args.unshift({ + type: "Bind", + args:[eP.portal, name, self.parameters, args] + }); + } + eP.on("Complete", function () { + if (exports.DEBUG > 0) { + sys.debug("Results length " + eP.results.length); + } + if (eP.results.length > 0 && eP.bound) { + if (exports.DEBUG > 0) { + sys.debug("Calling with results"); + } + // Callback gets wrapped at the tx layer, + // so we know when this gets tripped. + callback(eP.results); + } + else if (eP.noData === true) { + // Callback gets wrapped at the tx layer, + // so we know when this gets tripped. + callback(); + } + else { + eP.emit("error", "Complete reached without records or NoData state."); + } + }); + eP.on("BindComplete", function () { + eP.bound = true; + }); + eP.on("NoData", function () { + eP.noData = true; + }); + eP.on("newRow", function (row) { + eP.results.push(row); + }); if(exports.DEBUG > 0) { - sys.debug("Pushing execute message to tx in Prepared."); + sys.debug("Pushing execute message to eB in Prepared."); } - conn.push(eP); + eB.push(eP); } } Prepared.prototype = new message(); Prepared.prototype.constructor = Prepared; -function execute (prepared, params, args, row_desc, callback) { - var q = this; - var portal = md5(prepared + args.join("|")); // Clearly need a generated portal name here. - q.row_description = row_desc; - var results = []; - - var arr = [ - { - type: "Execute", - args: [portal, 0], // No limit. Get all the rows. - }, - { - type: "Flush", - args: [portal, 0] - }, - { - type: "Sync", - args: [] - } - ]; - - // If we have args, unshift the Bind. - if (args instanceof Array && args.length >= 1) { - arr.unshift({ - type: "Bind", - args:[portal, prepared, params, args], - callback: callback - }); - } - q.setMessages(arr); - - q.addListener("BindComplete", function (args) { - // We can mostly just ignore this, right? It's a notification of - // okay-ness. - conn.next(); - }); - - // Named this instead of DataRow to prevent the main loop from using this, - // instead of the main connection datarow parser. - - q.addListener("newRow", function (row) { - results.push(row); - }); - - q.addListener("CommandComplete", function (data) { - // Whatever we just did is ended. - // If it was a SELECT, args will be an array of rows, - // If it was an INSERT, etc, it'll be the type, and the number of - // affected rows. - if (exports.DEBUG > 0) { - sys.debug("Results length " + results.length); - } - if (results.length >= 1) { - if (exports.DEBUG > 0) { - sys.debug("Calling with results"); - } - // Callback gets wrapped at the tx layer, - // so we know when this gets tripped. - callback.call(q, results); - } - else { - this.type = data.type; - // Callback gets wrapped at the tx layer, - // so we know when this gets tripped. - callback.call(q); - } - }); -} +// function execute (prepared, args, callback) { +// var q = this; +// var prepared = prepared; +// // var +// q.row_description = row_desc; +// var results = []; +// +// var arr = [ +// { +// type: "Execute", +// args: [portal, 0], // No limit. Get all the rows. +// }, +// // { +// // type: "Flush", +// // args: [portal, 0] +// // }, +// // { +// // type: "Sync", +// // args: [] +// // } +// ]; +// +// // If we have args, unshift the Bind. +// if (args instanceof Array && args.length >= 1) { +// arr.unshift({ +// }); +// } +// q.setMessages(arr); +// +// q.addListener("BindComplete", function (args) { +// // We can mostly just ignore this, right? It's a notification of +// // okay-ness. +// // conn.next(); +// }); +// +// // Named this instead of DataRow to prevent the main loop from using this, +// // instead of the main connection datarow parser. +// +// q.addListener("newRow", function (row) { +// results.push(row); +// }); +// +// // This prevents the main command Complete from running, and cycling +// // to the next message in the TX buffer. +// // Crafty. +// +// q.addListener("Complete", function (data) { +// // Whatever we just did is ended. +// // If it was a SELECT, args will be an array of rows, +// // If it was an INSERT, etc, it'll be the type, and the number of +// // affected rows. +// +// }); +// } -execute.prototype = new message; +// execute.prototype = new message; function Sync (callback) { var q = this; @@ -727,7 +851,8 @@ function Connection(args) { if (current_tx === null ) { events.emit("nextTx"); } - else if (current_query === null || current_query === undefined || current_query.empty()) { + else if ( current_query === null || current_query === undefined || current_query.empty() === true ) { + sys.debug("Current query is empty. Moving to next query."); events.emit("nextQuery"); } else { @@ -761,7 +886,7 @@ function Connection(args) { sys.debug("nextMessage: Getting next message block"); events.emit("nextQuery"); } - else if (!current_query.empty()) { + else /* if ( current_query.empty() === false ) */ { var msg = current_query.next(); sys.debug(msg); if (msg && msg.type && msg.args) { @@ -772,15 +897,16 @@ function Connection(args) { readyState = false; sendMessage.apply(conn, [msg.type, msg.args]); } - else { - sys.debug("getting next query, nM"); - events.emit("nextQuery"); - } - } - else { - // wait - sys.debug("waiting for query buffer"); + // else { + // sys.debug("getting next query, nM"); + // events.emit("nextQuery"); + // } + sys.debug("waiting for RFQ"); } + // else { + // // wait + // + // } }); events.addListener("nextQuery", function () { @@ -865,7 +991,6 @@ function Connection(args) { case 23: // int4 value = parseInt(value, 10); break; - } } row[description.field] = value; @@ -883,8 +1008,8 @@ function Connection(args) { }); - events.addListener('CommandComplete', function (data) { - if (results.length >= 1) { + events.addListener('CommandComplete', function (data, results) { + if (results != null && results.length > 0) { // To allow for insert..returning current_query.emit("Complete", results, data); results = []; // blank the current result buffer. @@ -933,15 +1058,16 @@ function Connection(args) { var tx = conn.tx(); // automatically pushes onto the stack. var cb = null; if (typeof(callback) === 'function') { - cb = function (sth) { + cb = function () { // do the implicit sync/commit here? - callback(sth, tx); + callback.apply(callback, arguments); } if (exports.DEBUG > 0) { sys.debug("Transaction is: " + tx); } tx.prepare.call(tx, query, cb); events.emit("queryAdded"); + } else { // They didn't give us a prepared callback? That's.. odd. @@ -1054,23 +1180,13 @@ function Transaction (connection /*, params */) { }); // This acts effectively as a decorator from Python - wrap = function (func) { - return (function (rs) { - - func(rs); - callbackRun(); + var wrap = function (func) { + return (function () { + sys.debug(func); + func.apply(func, arguments); }); } - // Marks that we've run a callback, usually - // after the callback is completed. - // This is so we can track the lifecycle of this - // transaction, and test whether or not the connction - // can release and move on to the next buffered transaction. - callbackRun = function () { - message_length -= 1; - } - // The basic Query declaration. // This, by default, acts as a simple query when run without arguments // (allowing for certain queries to be handled faster), @@ -1104,7 +1220,7 @@ function Transaction (connection /*, params */) { var q = new Query(sql, function (rs) { sys.debug("RS is:" + sys.inspect(rs)); callback(rs); - callbackRun(); + // callbackRun(); }); if (exports.DEBUG > 0) { sys.debug("New plain query created: "+q); @@ -1115,17 +1231,17 @@ function Transaction (connection /*, params */) { // Standard prepared query. - this.prepare = function (sql, args, callback) { + this.prepare = function (sql, callback) { // Sets up a prepared query, and drops it onto the queue. if (sql.match(/\?/)) { var i = 1; sql = sql.replace(/\?/g, function () { return "$" + i++; }); } - thisp.push( - new Prepared(sql, thisp, wrap(callback)) - ); + var p = new Prepared(sql, thisp); + thisp.push(p); events.emit("queryAdded"); + wrap(callback)(p, thisp); // conn.emit.call(conn, "queryAdded"); } @@ -1160,11 +1276,7 @@ function Transaction (connection /*, params */) { } this.sync = function () { - thisp.push(new Sync( - function () { - callbackRun(); - } - )); + thisp.push(new Sync()); } this.push = function (msg) { From d118225497d04a803105d3a5c159cf4669ebec05 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Sat, 9 Oct 2010 23:33:16 -0700 Subject: [PATCH 30/54] Cleanup of the debug messages. --- demo.js | 2 +- lib/postgres-pure.js | 148 ++++++++++++++++++++----------------------- 2 files changed, 68 insertions(+), 82 deletions(-) diff --git a/demo.js b/demo.js index fc7f8a3..89b348b 100644 --- a/demo.js +++ b/demo.js @@ -1,6 +1,6 @@ var sys = require("sys"); var pg = require("./lib/postgres-pure"); -// pg.DEBUG=1; +pg.DEBUG=0; var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); db.query("SELECT 1::int as foobar;", function (rs, tx) { diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index a6518c5..9b28376 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -304,19 +304,27 @@ function message () { } } this.next = function() { - sys.debug("pos: " + pos); - sys.debug("messages:" + sys.inspect(self.messages)); + if (exports.DEBUG > 3) { + sys.debug("message pos: " + pos); + sys.debug("messages:" + sys.inspect(self.messages)); + } if (self.messages[pos] !== null && self.messages[pos] !== undefined) { pos = pos + 1; - sys.debug("Returning: " + self.messages[pos-1]); + if (exports.DEBUG > 3){ + sys.debug("Returning: " + self.messages[pos-1]); + } return self.messages[pos-1]; } - sys.debug("pos " + pos + " exceeds internal message buffer."); - sys.debug("Returning null"); + if (exports.DEBUG > 3) { + sys.debug("pos " + pos + " exceeds internal message buffer."); + sys.debug("Returning null"); + } return null; } this.empty = function () { - sys.debug("message pos " + pos + " exceeds length "+ self.messages.length +": " + (pos >= self.messages.length)); + if (exports.DEBUG > 3){ + sys.debug("message pos " + pos + " exceeds length "+ self.messages.length +": " + (pos >= self.messages.length)); + } if (pos >= self.messages.length) { return true; } @@ -336,6 +344,8 @@ function Query(sql, callback) { /* Returns the next query object in this object buffer. This can be null. + + Should there be a Sync message here? */ q.setMessages([ { @@ -351,13 +361,16 @@ function Query(sql, callback) { q.results.push(row); }); q.addListener("Complete", function (data) { - sys.debug("Callback: " + callback); + if (exports.DEBUG > 2) { + sys.debug("Callback: " + callback); + } + callback(q.results); }); q.toString = function () { return "Query: " + q.length}; q.addListener("RowDescription", function (desc) { q.row_description = desc; - if (exports.DEBUG > 0) { + if (exports.DEBUG > 2) { sys.debug("Caught RowDescription message."); } }); @@ -449,7 +462,9 @@ function Prepared(sql, conn /*, use_named */) { // execute command. if (currExec != null && currExec != undefined) { currExec.args.forEach(function (i) { - sys.debug("Arg is: " + i.type); + if (exports.DEBUG > 2) { + sys.debug("Arg is: " + i.type); + } self.addMessage(i); }); } @@ -473,8 +488,10 @@ function Prepared(sql, conn /*, use_named */) { // sys.debug("Message is: " + msg); // sys.debug("proto is: " + self.__proto__); // } - sys.debug("currexec " + currExec); - sys.debug("msg" + msg); + if (exports.DEBUG > 3) { + sys.debug("currexec " + currExec); + sys.debug("msg" + msg); + } if (msg !== null) { return msg; } @@ -486,9 +503,13 @@ function Prepared(sql, conn /*, use_named */) { if (eB[cPos] !== null && eB[cPos] !== undefined) { currExec = eB[cPos++]; // So we don't lose the reference to the // execute command. - sys.debug(sys.inspect(currExec.args.slice(1))); + if (exports.DEBUG > 3) { + sys.debug(sys.inspect(currExec.args.slice(1))); + } currExec.args.forEach(function (i) { - sys.debug("Arg is: " + i.type); + if (exports.DEBUG > 2) { + sys.debug("Arg is: " + i.type); + } self.addMessage(i); }); // for (arg in currExec.args.slice(1)) { @@ -501,11 +522,16 @@ function Prepared(sql, conn /*, use_named */) { // type: "Flush", // args: [], // }); // Flush the current query out. - sys.debug("Calling next from prototype."); + if (exports.DEBUG > 2) { + sys.debug("Calling next from prototype."); + } return self.__proto__.next(); } } - sys.debug("currExec isn't null, returning null from next"); + if (exports.DEBUG > 0) { + sys.debug("currExec isn't null, returning null from next"); + } + return null; // Patience.. } @@ -562,7 +588,7 @@ function Prepared(sql, conn /*, use_named */) { } if (eP.results.length > 0 && eP.bound) { if (exports.DEBUG > 0) { - sys.debug("Calling with results"); + sys.debug("Execute Complete: Calling with results"); } // Callback gets wrapped at the tx layer, // so we know when this gets tripped. @@ -595,62 +621,6 @@ function Prepared(sql, conn /*, use_named */) { Prepared.prototype = new message(); Prepared.prototype.constructor = Prepared; -// function execute (prepared, args, callback) { -// var q = this; -// var prepared = prepared; -// // var -// q.row_description = row_desc; -// var results = []; -// -// var arr = [ -// { -// type: "Execute", -// args: [portal, 0], // No limit. Get all the rows. -// }, -// // { -// // type: "Flush", -// // args: [portal, 0] -// // }, -// // { -// // type: "Sync", -// // args: [] -// // } -// ]; -// -// // If we have args, unshift the Bind. -// if (args instanceof Array && args.length >= 1) { -// arr.unshift({ -// }); -// } -// q.setMessages(arr); -// -// q.addListener("BindComplete", function (args) { -// // We can mostly just ignore this, right? It's a notification of -// // okay-ness. -// // conn.next(); -// }); -// -// // Named this instead of DataRow to prevent the main loop from using this, -// // instead of the main connection datarow parser. -// -// q.addListener("newRow", function (row) { -// results.push(row); -// }); -// -// // This prevents the main command Complete from running, and cycling -// // to the next message in the TX buffer. -// // Crafty. -// -// q.addListener("Complete", function (data) { -// // Whatever we just did is ended. -// // If it was a SELECT, args will be an array of rows, -// // If it was an INSERT, etc, it'll be the type, and the number of -// // affected rows. -// -// }); -// } - -// execute.prototype = new message; function Sync (callback) { var q = this; @@ -842,7 +812,7 @@ function Connection(args) { // We can cycle to the next? - if (exports.DEBUG > 0) { + if (exports.DEBUG > 3) { sys.debug("Readystate is: " + readyState); sys.debug("TX Queue length: "+tx_queue.length); sys.debug("TX is "+ current_tx); @@ -852,7 +822,9 @@ function Connection(args) { events.emit("nextTx"); } else if ( current_query === null || current_query === undefined || current_query.empty() === true ) { - sys.debug("Current query is empty. Moving to next query."); + if (exports.DEBUG > 3) { + sys.debug("3: Current query is empty. Moving to next query."); + } events.emit("nextQuery"); } else { @@ -883,12 +855,18 @@ function Connection(args) { // If it's null, it calls nextQuery. if (current_query === null) { - sys.debug("nextMessage: Getting next message block"); + if (exports.DEBUG >1 ) { + sys.debug("nextMessage: Getting next message block"); + } + events.emit("nextQuery"); } else /* if ( current_query.empty() === false ) */ { var msg = current_query.next(); - sys.debug(msg); + if (exports.DEBUG > 3) { + sys.debug(msg); + } + if (msg && msg.type && msg.args) { // We have what we need to perform this query. if (exports.DEBUG > 0) { @@ -901,7 +879,9 @@ function Connection(args) { // sys.debug("getting next query, nM"); // events.emit("nextQuery"); // } - sys.debug("waiting for RFQ"); + if (exports.DEBUG > 2){ + sys.debug("waiting for RFQ"); + } } // else { // // wait @@ -1182,7 +1162,9 @@ function Transaction (connection /*, params */) { // This acts effectively as a decorator from Python var wrap = function (func) { return (function () { - sys.debug(func); + if (exports.DEBUG > 3) { + sys.debug(func); + } func.apply(func, arguments); }); } @@ -1199,7 +1181,7 @@ function Transaction (connection /*, params */) { /* sql, some_args, callback */ this.query = function () { var args = Array.prototype.slice.call(arguments); - if (exports.DEBUG>0) { + if (exports.DEBUG>3) { sys.debug("Args are: " + args); } var sql = args.shift(); @@ -1218,7 +1200,9 @@ function Transaction (connection /*, params */) { // This does not require a normal Sync message // or any other such magic-ery. var q = new Query(sql, function (rs) { - sys.debug("RS is:" + sys.inspect(rs)); + if (exports.DEBUG > 3) { + sys.debug("RS is:" + sys.inspect(rs)); + } callback(rs); // callbackRun(); }); @@ -1297,7 +1281,9 @@ function Transaction (connection /*, params */) { return messages.shift(); // Front of the array, there. } } - sys.debug("Returning null from tx.next()"); + if (exports.DEBUG > 0){ + sys.debug("tx: Returning null from tx.next()"); + } return null; } } From 99f9bf66d8c66d3bc7f528716dc687f3a8b19e39 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Mon, 11 Oct 2010 14:53:22 -0700 Subject: [PATCH 31/54] Adding license --- LICENSE | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..eccf303 --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2010 Tim Caswell , + (c) 2010 Aurynn Shaw + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file From 2010a272595c7124ac7f76a2fe349b3c51f078f7 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Fri, 15 Oct 2010 13:31:18 -0700 Subject: [PATCH 32/54] Fixed a bug with prepared queries not obtaining a localized scope. Added some initial test cases under t/ Initial test work on the transaction object --- demo.js | 33 +++++----- lib/postgres-pure.js | 137 ++++++++++++------------------------------ t/complex_prepared.js | 33 ++++++++++ t/query.js | 12 ++++ t/simple_prepared.js | 15 +++++ 5 files changed, 119 insertions(+), 111 deletions(-) create mode 100644 t/complex_prepared.js create mode 100644 t/query.js create mode 100644 t/simple_prepared.js diff --git a/demo.js b/demo.js index 89b348b..65feeef 100644 --- a/demo.js +++ b/demo.js @@ -12,26 +12,31 @@ db.query("SELECT 1::int as foobar;", function (rs, tx) { }); db.prepare("SELECT ?::int AS foobar", function (sth, tx) { - sth.execute(1, function (rs, tx) { + sth.execute(1, function (rs) { sys.puts(sys.inspect(rs)); }); sth.execute(2, function (rs) { sys.puts(sys.inspect(rs)); }); + tx.prepare("SELECT ?::int AS cheese", function (sth) { + sth.execute(3, function (rs) { + sys.puts(sys.inspect(rs)); + }) ; + }); // db.close(); }); -db.close(); -// -// db.transaction(function (tx) { -// tx.query("SELECT ?::int AS foobar", 1, function (rs) { -// sys.puts(sys.inspect(rs)); -// }); -// tx.prepare("SELECT ?::int AS foobar", function (sth) { -// sth.execute(2, function (rs) { -// sys.puts(sys.inspect(rs)); -// }); -// }); -// tx.commit(); -// }); \ No newline at end of file +db.transaction(function (tx) { + tx.query("SELECT ?::int AS txtest1", 1, function (rs) { + sys.puts(sys.inspect(rs)); + }); + tx.prepare("SELECT ?::int AS txtest2", function (sth) { + sth.execute(2, function (rs) { + sys.puts(sys.inspect(rs)); + }); + }); + tx.commit(); +}); + +db.close(); \ No newline at end of file diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 9b28376..5d4420c 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -289,8 +289,9 @@ function parse_response(code, buffer) { function message () { var self = this; - this.messages = []; - var pos = 0; // instance variable + this.messages = []; // public instance array. + var pos = 0; + this._position = 0; // instance variable this.setMessages = function (msg) { if (msg instanceof Array) { self.messages = msg; @@ -305,27 +306,27 @@ function message () { } this.next = function() { if (exports.DEBUG > 3) { - sys.debug("message pos: " + pos); - sys.debug("messages:" + sys.inspect(self.messages)); + sys.debug("message pos: " + this._position); + sys.debug("messages:" + sys.inspect(this.messages)); } - if (self.messages[pos] !== null && self.messages[pos] !== undefined) { - pos = pos + 1; + if (this.messages[this._position] !== null && this.messages[this._position] !== undefined) { + this._position = this._position + 1; if (exports.DEBUG > 3){ - sys.debug("Returning: " + self.messages[pos-1]); + sys.debug("Returning: " + this.messages[this._position-1]); } - return self.messages[pos-1]; + return this.messages[this._position-1]; } if (exports.DEBUG > 3) { - sys.debug("pos " + pos + " exceeds internal message buffer."); + sys.debug("pos " + this._position + " exceeds internal message buffer."); sys.debug("Returning null"); } return null; } this.empty = function () { if (exports.DEBUG > 3){ - sys.debug("message pos " + pos + " exceeds length "+ self.messages.length +": " + (pos >= self.messages.length)); + sys.debug("message pos " + this._position + " exceeds length "+ this.messages.length +": " + (this._position >= this.messages.length)); } - if (pos >= self.messages.length) { + if (this._position >= this.messages.length) { return true; } return false; @@ -333,7 +334,7 @@ function message () { } message.prototype = new process.EventEmitter; - +// message.prototype.constructor = message; function Query(sql, callback) { message.call(this); @@ -376,7 +377,7 @@ function Query(sql, callback) { }); } -Query.prototype = new message(); +Query.prototype = new message; Query.prototype.constructor = Query; function Prepared(sql, conn /*, use_named */) { @@ -386,10 +387,11 @@ function Prepared(sql, conn /*, use_named */) { // var prepared_name = md5(sql); // Use the md5 hash. This is easily selectable later. var self = this; - // message.call(this); + message.call(this); self.row_description = null; self.parameters = null; self.noData = false; + self.sql = sql; var parseComplete = null; var readyToExec = false; @@ -402,7 +404,7 @@ function Prepared(sql, conn /*, use_named */) { type: "Parse", // Prepared name, the query, // and a zero-length array to declare no types. - args: ['', sql, []], + args: ['', self.sql, []], }, { type: "Describe", @@ -483,11 +485,7 @@ function Prepared(sql, conn /*, use_named */) { self.next = function () { // Override. - var msg = this.__proto__.next(); - // if ( exports.DEBUG > 0) { - // sys.debug("Message is: " + msg); - // sys.debug("proto is: " + self.__proto__); - // } + var msg = this.__proto__.next.call(this); if (exports.DEBUG > 3) { sys.debug("currexec " + currExec); sys.debug("msg" + msg); @@ -508,24 +506,14 @@ function Prepared(sql, conn /*, use_named */) { } currExec.args.forEach(function (i) { if (exports.DEBUG > 2) { - sys.debug("Arg is: " + i.type); + sys.debug("Adding Arg: " + i.type); } - self.addMessage(i); + self.addMessage.call(self, i); }); - // for (arg in currExec.args.slice(1)) { - // - // if (currExec.args.hasOwnProperty(arg)) { - // - // } - // } - // self.addMessage({ - // type: "Flush", - // args: [], - // }); // Flush the current query out. if (exports.DEBUG > 2) { sys.debug("Calling next from prototype."); } - return self.__proto__.next(); + return self.__proto__.next.call(this); } } if (exports.DEBUG > 0) { @@ -1042,8 +1030,9 @@ function Connection(args) { // do the implicit sync/commit here? callback.apply(callback, arguments); } - if (exports.DEBUG > 0) { + if (exports.DEBUG > 3) { sys.debug("Transaction is: " + tx); + sys.debug("Query is: " + query); } tx.prepare.call(tx, query, cb); events.emit("queryAdded"); @@ -1118,16 +1107,19 @@ function Connection(args) { Ergo, we set up a DB listener with the same name, and fire our emitter when it's triggered. - Easy. + Easy, and very nifty. */ conn.addListener('newListener', function (e, listener) { if (e === 'String') { // It's a string. - if (!(e in ['newListener'])) - conn.notify(e, listener); + if (!(e in ['newListener'])) { + conn.notify(e, listener); + } } - }); + conn.notify = function (name) { + + } } Connection.prototype = new process.EventEmitter(); @@ -1220,12 +1212,15 @@ function Transaction (connection /*, params */) { // Sets up a prepared query, and drops it onto the queue. if (sql.match(/\?/)) { var i = 1; - sql = sql.replace(/\?/g, function () { return "$" + i++; }); + var fsql = sql.replace(/\?/g, function () { return "$" + i++; }); } - var p = new Prepared(sql, thisp); + var p = new Prepared(fsql, thisp); thisp.push(p); events.emit("queryAdded"); wrap(callback)(p, thisp); + if (exports.DEBUG == 4) { + sys.debug("Prepared messages: " + sys.inspect(messages)); + } // conn.emit.call(conn, "queryAdded"); } @@ -1244,7 +1239,7 @@ function Transaction (connection /*, params */) { this.begin = function () { // Begins the transaction. We now lock the transaction to the wire. - thisp.append(thisp.query("BEGIN;")); // Don't need to watch for callbacks. + thisp.push(thisp.query("BEGIN;")); // Don't need to watch for callbacks. } this.rollback = function () { // Rolls back the request, and does the connection release. @@ -1278,6 +1273,9 @@ function Transaction (connection /*, params */) { this.next = function () { if (messages.length > 0) { if (messages[0] !== null && messages[0] !== undefined) { + if (exports.DEBUG == 4) { + sys.debug("TX Returning: " + sys.inspect(messages[0])); + } return messages.shift(); // Front of the array, there. } } @@ -1295,59 +1293,4 @@ function connectionManager (dsn /*, connections=1 */) { // var conn = new Connection(dsn); } -exports.connect = Connection; - - - -/* Allows a currently-executing query to selectively modify the current - query queue, IF it is currently executing. - - Otherwise, it splices the query object in after its position in the queue. -*/ -// conn.yield_to = function (first, query) { -// if (exports.DEBUG > 0) { -// sys.debug("got yield_to"); -// } -// if (first === current_query) { -// query_queue.unshift(query); -// events.emit("queryAdded"); // Don't immediately switch to the next message. -// } -// else if (first in query_queue) { -// // Splice it in after the query -// query_queue.splice( query_queue.indexOf(first), 0, query ); -// events.emit("queryAdded"); -// } -// } -// // Transactions shouldn't release themselves, I don't think. -// // -// conn.release = function (query) { -// if (exports.DEBUG > 0) { -// sys.debug("got release"); -// } -// if (query === current_query) { -// readyState = true; -// events.emit("ReadyForQuery"); // Cycle along. -// } -// } - - -// var parameters, callback; -// -// // Grab the variable length parameters and the row_callback is there is one. -// parameters = Array.prototype.slice.call(arguments, 1); -// -// if (typeof parameters[parameters.length - 1] === 'function') { -// callback = parameters.pop(); -// } -// var q; -// if (parameters.length == 1 && parameters[0] instanceof Array) { -// // We have a parameterized query -// q = new Prepared(query, function (sth) { -// sth.execute(parameters[0], callback); -// }); -// } -// else { -// q = new Query(query, callback); -// -// } -// query_queue.push(q); \ No newline at end of file +exports.connect = Connection; \ No newline at end of file diff --git a/t/complex_prepared.js b/t/complex_prepared.js new file mode 100644 index 0000000..530065b --- /dev/null +++ b/t/complex_prepared.js @@ -0,0 +1,33 @@ +var sys = require("sys"); +var pg = require("../lib/postgres-pure"); +pg.DEBUG=0; + +var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); + +db.prepare("SELECT ?::int AS foobar", function (sth, tx) { + sth.execute(1, function (rs) { + console.log(eq(rs[0]['foobar'], 1)); + //sys.puts(sys.inspect(rs)); + }); + sth.execute(2, function (rs) { + console.log(eq(rs[0]['foobar'], 2)); + // sys.puts(sys.inspect(rs)); + + }); + tx.prepare("SELECT ?::int AS cheese", function (sth) { + sth.execute(3, function (rs) { + console.log(eq(rs[0]['cheese'], 3)); + // sys.puts(sys.inspect(rs)); + }); + }); + // db.close(); +}); +db.close(); + + +function eq (l, r) { + if (l === r) { + return "ok" + } + return "not ok\n " + l + " != " + r; +} \ No newline at end of file diff --git a/t/query.js b/t/query.js new file mode 100644 index 0000000..0625288 --- /dev/null +++ b/t/query.js @@ -0,0 +1,12 @@ +var sys = require("sys"); +var pg = require("../lib/postgres-pure"); +pg.DEBUG=0; + +var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); +db.query("SELECT 1::int as foobar;", function (rs, tx) { + sys.puts(sys.inspect(rs)); + tx.query("SELECT 2::int as foobartwo", function (rs) { + sys.puts(sys.inspect(rs)); + }); +}); +db.close(); \ No newline at end of file diff --git a/t/simple_prepared.js b/t/simple_prepared.js new file mode 100644 index 0000000..ac3b814 --- /dev/null +++ b/t/simple_prepared.js @@ -0,0 +1,15 @@ +var sys = require("sys"); +var pg = require("../lib/postgres-pure"); +pg.DEBUG=0; + +var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); +db.prepare("SELECT ?::int AS foobar", function (sth, tx) { + sth.execute(1, function (rs) { + sys.puts(sys.inspect(rs)); + }); + sth.execute(2, function (rs) { + sys.puts(sys.inspect(rs)); + + }); +}); +db.close(); From 79166de4a60d1db9d9a20fc6d0a3b1e3830ca908 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Fri, 15 Oct 2010 13:40:36 -0700 Subject: [PATCH 33/54] Transaction support now works As Expected. TX objects do not (yet) support implicit BEGIN. --- demo.js | 3 ++- lib/postgres-pure.js | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/demo.js b/demo.js index 65feeef..1558ebe 100644 --- a/demo.js +++ b/demo.js @@ -28,6 +28,7 @@ db.prepare("SELECT ?::int AS foobar", function (sth, tx) { }); db.transaction(function (tx) { + // tx.begin(); tx.query("SELECT ?::int AS txtest1", 1, function (rs) { sys.puts(sys.inspect(rs)); }); @@ -36,7 +37,7 @@ db.transaction(function (tx) { sys.puts(sys.inspect(rs)); }); }); - tx.commit(); + // tx.commit(); }); db.close(); \ No newline at end of file diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 5d4420c..70f9a4d 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -909,8 +909,12 @@ function Connection(args) { } if (readyState) { if (tx_queue.length > 0) { + sys.debug("nextTx: tx_queue length is: " + tx_queue.length); current_tx = tx_queue.shift(); if (current_tx !== null && current_tx !== undefined) { + if (exports.DEBUG>2) { + sys.debug("nextTx: current_tx is " + sys.inspect(current_tx)); + } events.emit("nextQuery"); } else { @@ -1036,7 +1040,6 @@ function Connection(args) { } tx.prepare.call(tx, query, cb); events.emit("queryAdded"); - } else { // They didn't give us a prepared callback? That's.. odd. @@ -1047,14 +1050,16 @@ function Connection(args) { // Sets up a new transaction object/block. // Unsure if this should do an implicit BEGIN. // Maybe a setting? - this.transaction = function () { - + this.transaction = function (callback) { var tx = new Transaction(this); tx.on("ranCallback", function () { if (exports.DEBUG >0 ) { sys.debug("Received notification of callback execution."); } }); + if (callback !== null && callback !== undefined) { + callback(tx); + } conn.push(tx); return tx; } From b1372063f66a870984931a4835a9a393df75720c Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Fri, 15 Oct 2010 13:47:13 -0700 Subject: [PATCH 34/54] Re-removing the debugging line. --- demo.js | 1 - lib/postgres-pure.js | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/demo.js b/demo.js index 1558ebe..c0f1b4c 100644 --- a/demo.js +++ b/demo.js @@ -39,5 +39,4 @@ db.transaction(function (tx) { }); // tx.commit(); }); - db.close(); \ No newline at end of file diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 70f9a4d..e3bc6cf 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -909,7 +909,9 @@ function Connection(args) { } if (readyState) { if (tx_queue.length > 0) { - sys.debug("nextTx: tx_queue length is: " + tx_queue.length); + if (exports.DEBUG > 1) { + sys.debug("nextTx: tx_queue length is: " + tx_queue.length); + } current_tx = tx_queue.shift(); if (current_tx !== null && current_tx !== undefined) { if (exports.DEBUG>2) { From 7abae81481a0f7d9b3f415420664f01b424d28d1 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Fri, 15 Oct 2010 13:52:05 -0700 Subject: [PATCH 35/54] Update the README with more accurate/complete documentation. --- README.md | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index f818889..c4d6f57 100644 --- a/README.md +++ b/README.md @@ -3,31 +3,33 @@ This library is a implementation of the PostgreSQL backend/frontend protocol in javascript. It uses the node.js tcp and event libraries. A javascript md5 library is included for servers that require md5 password hashing (this is default). -This library allows for the correct handling of prepared queries. +This library allows for the correct handling of server-side prepared queries. -If you wish to nest DB calls, db.close must be in the deepest callback, or all statements that occur inside of callbacks deeper than the callback which handles db.close will not be executed. +Nested DB calls will be executed in the order of definition. All code is available under the terms of the MIT license, unless otherwise noted (md5.js) +(c) 2010, Tim Caswell, Aurynn Shaw. + ## Example use var sys = require("sys"); - var pg = require("postgres"); + var pg = require("postgres-pure"); var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); db.query("SELECT * FROM sometable", function (data) { - sys.p(data); + console.log(data); }); db.close(); ## Example use of Parameterized Queries var sys = require("sys"); - var pg = require("postgres"); + var pg = require("postgres-pure"); var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); - db.query("SELECT * FROM yourtable WHERE id = ?", [1], function (data) { - sys.p(data); + db.query("SELECT * FROM yourtable WHERE id = ?", 1, function (data) { + console.log(data); }); db.close(); @@ -37,10 +39,9 @@ All code is available under the terms of the MIT license, unless otherwise noted var pg = require("postgres"); var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); - - var stmt = db.prepare("SELECT * FROM yourtable WHERE id = ?"); - - stmt.execute([1], function (d) { - sys.p(d); - db.close(); - }); \ No newline at end of file + db.prepare("SELECT * FROM yourtable WHERE id = ?", function (stmt) { + stmt.execute(1, function (rs) { + console.log(rs[0]); + }); + }); + db.close(); \ No newline at end of file From a99f3367458f6315c90bb65b9bdd7e7a6e8b8e89 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Thu, 21 Oct 2010 21:14:42 -0700 Subject: [PATCH 36/54] Adding support for INSERT...RETURNING Additional support for NoData queries. They should now run correctly and not crash the driver. TODO: Provide a null result set and a TX object that knows how many rows were modified (if any) for insert, etc. commands. --- README.md | 2 + demo.js | 82 ++++++++++++++++++++++++---------------- lib/postgres-pure.js | 89 +++++++++++++++++++++++++++++--------------- 3 files changed, 109 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index c4d6f57..230ddf1 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ All code is available under the terms of the MIT license, unless otherwise noted (c) 2010, Tim Caswell, Aurynn Shaw. +Bugs can be reported @ https://public.commandprompt.com/projects/postgres-js + ## Example use var sys = require("sys"); diff --git a/demo.js b/demo.js index c0f1b4c..d88f96d 100644 --- a/demo.js +++ b/demo.js @@ -2,41 +2,57 @@ var sys = require("sys"); var pg = require("./lib/postgres-pure"); pg.DEBUG=0; -var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); -db.query("SELECT 1::int as foobar;", function (rs, tx) { - sys.puts(sys.inspect(rs)); - tx.query("SELECT 2::int as foobartwo", function (rs) { - sys.puts(sys.inspect(rs)); - }); - // db.close(); -}); +var db = new pg.connect("pgsql://test:12345@localhost:5432/pdxpugtest"); +// db.query("explain analyze select * from pg_class ;", function (rs, tx) { +// sys.puts(sys.inspect(rs)); +// // tx.query("SELECT 2::int as querytest2", function (rs) { +// // sys.puts(sys.inspect(rs)); +// // }); +// }); -db.prepare("SELECT ?::int AS foobar", function (sth, tx) { - sth.execute(1, function (rs) { - sys.puts(sys.inspect(rs)); - }); - sth.execute(2, function (rs) { - sys.puts(sys.inspect(rs)); +db.prepare("INSERT INTO pdxpug (id) VALUES (?) RETURNING id", function (sth) { + sth.execute(1, function(rs) { + if (rs === undefined) { + console.log("No data."); + } + else { + console.log(sys.inspect(rs)); + } }); - tx.prepare("SELECT ?::int AS cheese", function (sth) { - sth.execute(3, function (rs) { - sys.puts(sys.inspect(rs)); - }) ; - }); - // db.close(); }); -db.transaction(function (tx) { - // tx.begin(); - tx.query("SELECT ?::int AS txtest1", 1, function (rs) { - sys.puts(sys.inspect(rs)); - }); - tx.prepare("SELECT ?::int AS txtest2", function (sth) { - sth.execute(2, function (rs) { - sys.puts(sys.inspect(rs)); - }); - }); - // tx.commit(); -}); -db.close(); \ No newline at end of file +// db.prepare().on("some_event"); + +// db.prepare("SELECT ?::int AS preparetest", function (sth, tx) { +// sth.execute(1, function (rs) { +// sys.puts(sys.inspect(rs)); +// }); +// sth.execute(2, function (rs) { +// sys.puts(sys.inspect(rs)); +// +// }); +// // tx.prepare("SELECT ?::int AS preparetest2", function (sth) { +// // sth.execute(3, function (rs) { +// // sys.puts(sys.inspect(rs)); +// // }) ; +// // }); +// }); + +// db.transaction(function (tx) { +// // tx.begin(); +// tx.query("SELECT ?::int AS txtest1", 1, function (rs) { +// sys.puts(sys.inspect(rs)); +// }); +// tx.prepare("SELECT ?::int AS txtest2", function (sth) { +// sth.execute(2, function (rs) { +// sys.puts(sys.inspect(rs)); +// }); +// }); +// // tx.commit(); +// }); +db.close(); + +// db.prepare(query, function (sth, tx) { +// sth.execute(args, callback, errback); +// }) \ No newline at end of file diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index e3bc6cf..6053247 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -331,12 +331,22 @@ function message () { } return false; } + this.on("Error", function (err) { + if (errback !== null && errback !== undefined) { + errback(err); + } + else { + // We don't have an error handler? Odd.. + // Pass it up to our connection/transaction. + conn.emit("Error", err); + } + }); } message.prototype = new process.EventEmitter; // message.prototype.constructor = message; -function Query(sql, callback) { +function Query(sql, callback, errback) { message.call(this); this.sql = sql; var q = this; @@ -358,10 +368,10 @@ function Query(sql, callback) { // args: [] // } ]); - q.addListener("newRow", function (row) { + q.on("newRow", function (row) { q.results.push(row); }); - q.addListener("Complete", function (data) { + q.on("Complete", function (data) { if (exports.DEBUG > 2) { sys.debug("Callback: " + callback); } @@ -369,7 +379,7 @@ function Query(sql, callback) { callback(q.results); }); q.toString = function () { return "Query: " + q.length}; - q.addListener("RowDescription", function (desc) { + q.on("RowDescription", function (desc) { q.row_description = desc; if (exports.DEBUG > 2) { sys.debug("Caught RowDescription message."); @@ -418,7 +428,7 @@ function Prepared(sql, conn /*, use_named */) { self.setMessages(arr); - self.addListener("ParseComplete", function () { + self.on("ParseComplete", function () { // Execute can now be run successfully. // Until this point, we can't assume that there's a matching query. // Anyway, we now run a DESCRIBE operation, and store the row @@ -441,9 +451,6 @@ function Prepared(sql, conn /*, use_named */) { self.parameters = desc; }); - self.on("NoData", function () { - self.noData = true; - }) self.on("newRow", function (row) { currExec.emit("newRow", row); }); @@ -473,6 +480,12 @@ function Prepared(sql, conn /*, use_named */) { } }); + self.on("NoData", function () { + if (currExec !== null) { + currExec.noData = true; + } + }); + self.on("BindComplete", function () { if (currExec != null) { currExec.emit("BindComplete"); @@ -574,6 +587,7 @@ function Prepared(sql, conn /*, use_named */) { if (exports.DEBUG > 0) { sys.debug("Results length " + eP.results.length); } + sys.debug("noData is: "+eP.noData); if (eP.results.length > 0 && eP.bound) { if (exports.DEBUG > 0) { sys.debug("Execute Complete: Calling with results"); @@ -595,6 +609,7 @@ function Prepared(sql, conn /*, use_named */) { eP.bound = true; }); eP.on("NoData", function () { + eP.noData = true; }); eP.on("newRow", function (row) { @@ -605,6 +620,9 @@ function Prepared(sql, conn /*, use_named */) { } eB.push(eP); } + self.on("Error", function (err) { + + }); } Prepared.prototype = new message(); Prepared.prototype.constructor = Prepared; @@ -754,34 +772,43 @@ function Connection(args) { } // Set up tcp client - connection.addListener("connect", function () { + connection.on("connect", function () { sendMessage('StartupMessage', [{user: args.username, database: args.database}]); }); - connection.addListener("data", function (data) { + connection.on("data", function (data) { if (exports.DEBUG > 2) { sys.debug("<-" + data.inspect()); } queue.push(data); checkInput(); }); - connection.addListener("end", function (data) { + connection.on("end", function (data) { connection.end(); }); - connection.addListener("disconnect", function (had_error) { + connection.on("disconnect", function (had_error) { if (had_error) { sys.debug("CONNECTION DIED WITH ERROR"); } }); // Set up callbacks to automatically do the login and other logic - events.addListener('AuthenticationMD5Password', function (salt) { + events.on('AuthenticationMD5Password', function (salt) { var result = "md5" + md5(md5(args.password + args.username) + salt.toString("binary")); sendMessage('PasswordMessage', [result]); }); - events.addListener('AuthenticationCleartextPassword', function () { + events.on('AuthenticationCleartextPassword', function () { sendMessage('PasswordMessage', [args.password]); }); - events.addListener('ErrorResponse', function (e) { + + /* + Errors shuld be handled at the TX layer. + This allows for a given TX set (which contains 1..n message sets) to + catch and gracefully recover from an error state, and not attempt to + continue to slam messages onto the wire when there isn't an ability to + do so. + */ + + events.on('ErrorResponse', function (e) { conn.emit('error', e.S + ": " + e.M); if (e.S === 'FATAL') { connection.end(); @@ -863,21 +890,13 @@ function Connection(args) { readyState = false; sendMessage.apply(conn, [msg.type, msg.args]); } - // else { - // sys.debug("getting next query, nM"); - // events.emit("nextQuery"); - // } if (exports.DEBUG > 2){ sys.debug("waiting for RFQ"); } } - // else { - // // wait - // - // } }); - events.addListener("nextQuery", function () { + events.on("nextQuery", function () { if (exports.DEBUG > 0) { sys.debug("got nextQuery"); } @@ -936,7 +955,7 @@ function Connection(args) { // This should always be caught by the current query. - events.addListener("RowDescription", function (data) { + events.on("RowDescription", function (data) { row_description = data; results = []; }); @@ -945,7 +964,7 @@ function Connection(args) { // Data row is handled by the connection for the time // being, even though we should be looking at handling it in the // query object, where the RowDescription lives. - events.addListener("DataRow", function (data) { + events.on("DataRow", function (data) { var row, i, l, description, value; row = {}; l = data.length; @@ -982,7 +1001,7 @@ function Connection(args) { }); - events.addListener('CommandComplete', function (data, results) { + events.on('CommandComplete', function (data, results) { if (results != null && results.length > 0) { // To allow for insert..returning current_query.emit("Complete", results, data); @@ -1116,7 +1135,7 @@ function Connection(args) { Easy, and very nifty. */ - conn.addListener('newListener', function (e, listener) { + conn.on('newListener', function (e, listener) { if (e === 'String') { // It's a string. if (!(e in ['newListener'])) { @@ -1162,7 +1181,7 @@ function Transaction (connection /*, params */) { var wrap = function (func) { return (function () { if (exports.DEBUG > 3) { - sys.debug(func); + sys.debug("Wrapping function: " + func); } func.apply(func, arguments); }); @@ -1226,7 +1245,7 @@ function Transaction (connection /*, params */) { events.emit("queryAdded"); wrap(callback)(p, thisp); if (exports.DEBUG == 4) { - sys.debug("Prepared messages: " + sys.inspect(messages)); + sys.debug("Prepared messages: " + sys.inspect(messages, 4)); } // conn.emit.call(conn, "queryAdded"); } @@ -1270,7 +1289,7 @@ function Transaction (connection /*, params */) { messages.push(msg); this.emit("queryAdded"); if (exports.DEBUG > 0) { - sys.debug("Added message of " + msg + " to TX"); + sys.debug("Added message of " + sys.inspect(msg) + " to TX"); } } else { @@ -1291,6 +1310,14 @@ function Transaction (connection /*, params */) { } return null; } + this.on("Error", function (e) { + /* Global transaction error response. + This should push a SYNC message into the buffer immediately, + as well as a ROLLBACK command. + This will free up the wire from whatever the last message set was, + and allow for a given piece of code to recover gracefully. + */ + }); } Transaction.prototype = new process.EventEmitter(); From fa7aa22e08bfe84ed09c9ece547119f79fb4c373 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Wed, 27 Oct 2010 21:07:02 -0700 Subject: [PATCH 37/54] Update to support the fully expanded, node-style error response pattern, specifically: db.query(query, args, function (error, results){}); --- demo.js | 29 ++++++++++------- lib/postgres-pure.js | 75 ++++++++++++++++++++++++++++++-------------- 2 files changed, 70 insertions(+), 34 deletions(-) diff --git a/demo.js b/demo.js index d88f96d..f811fd8 100644 --- a/demo.js +++ b/demo.js @@ -1,24 +1,31 @@ var sys = require("sys"); var pg = require("./lib/postgres-pure"); -pg.DEBUG=0; +// pg.DEBUG=4; -var db = new pg.connect("pgsql://test:12345@localhost:5432/pdxpugtest"); -// db.query("explain analyze select * from pg_class ;", function (rs, tx) { -// sys.puts(sys.inspect(rs)); -// // tx.query("SELECT 2::int as querytest2", function (rs) { -// // sys.puts(sys.inspect(rs)); -// // }); -// }); +var db = new pg.connect("pgsql://test:12345@localhost:5432/returning_test"); +db.query("SELECT 1::querytest;", function (error, rs, tx) { + if (error) { + console.log("Error!"); + sys.puts(error); + sys.puts(error.code); + } + else { + sys.puts(sys.inspect(rs)); + } + + // tx.query("SELECT 2::int as querytest2", function (rs) { + // sys.puts(sys.inspect(rs)); + // }); +}); -db.prepare("INSERT INTO pdxpug (id) VALUES (?) RETURNING id", function (sth) { - sth.execute(1, function(rs) { +db.prepare("INSERT INTO returning_test (val) VALUES (?) RETURNING id, val", function (sth) { + sth.execute("text value", function(e, rs) { if (rs === undefined) { console.log("No data."); } else { console.log(sys.inspect(rs)); } - }); }); diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 6053247..d653121 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -199,10 +199,18 @@ function parse_response(code, buffer) { break; case 'E': type = "ErrorResponse"; - args = [{}]; + var err = {}; + // args = [{}]; reader.multicstring().forEach(function (field) { - args[0][field[0]] = field.substr(1); + err[field[0]] = field.substr(1); }); + // Now, convert it to a more useful object. + var obj = new Error(); + obj.code = err['C']; + obj.message = err['M']; + obj.severity = err['S']; + + args = [obj]; break; case 't': type = "ParameterDescription", @@ -286,6 +294,14 @@ function parse_response(code, buffer) { return {type: type, args: args}; } +function Error () { + this.severity = ""; + this.code = ""; + this.message = ""; + this.toString = function () { + return "[postgres.js] " + this.severity + ": " + this.message; + } +} function message () { var self = this; @@ -346,7 +362,7 @@ function message () { message.prototype = new process.EventEmitter; // message.prototype.constructor = message; -function Query(sql, callback, errback) { +function Query(sql, callback) { message.call(this); this.sql = sql; var q = this; @@ -376,7 +392,7 @@ function Query(sql, callback, errback) { sys.debug("Callback: " + callback); } - callback(q.results); + callback(null, q.results); }); q.toString = function () { return "Query: " + q.length}; q.on("RowDescription", function (desc) { @@ -385,6 +401,10 @@ function Query(sql, callback, errback) { sys.debug("Caught RowDescription message."); } }); + q.on("ErrorResponse", function (e) { + callback(e); + }); + } Query.prototype = new message; @@ -492,6 +512,15 @@ function Prepared(sql, conn /*, use_named */) { } }); + self.on("ErrorResponse", function (err) { + if (currExec != null) { + currExec.emit("ErrorResponse", err); + } + else { + self.error = e; + } + }); + // self.empty = function () { // return self.__proto__.empty.call(self); // } @@ -594,7 +623,7 @@ function Prepared(sql, conn /*, use_named */) { } // Callback gets wrapped at the tx layer, // so we know when this gets tripped. - callback(eP.results); + callback(null, eP.results); } else if (eP.noData === true) { // Callback gets wrapped at the tx layer, @@ -615,14 +644,14 @@ function Prepared(sql, conn /*, use_named */) { eP.on("newRow", function (row) { eP.results.push(row); }); + eP.on("ErrorResponse", function (e) { + callback(e); + }); if(exports.DEBUG > 0) { sys.debug("Pushing execute message to eB in Prepared."); } eB.push(eP); } - self.on("Error", function (err) { - - }); } Prepared.prototype = new message(); Prepared.prototype.constructor = Prepared; @@ -1025,10 +1054,10 @@ function Connection(args) { var callback = args.pop(); if (typeof(callback) === 'function') { - args.push(function (rs) { + args.push(function (err, rs) { // do the implicit sync/commit here? // tx.commit(); - callback(rs, tx); // So they have access to the transaction? + callback(err, rs, tx); // So they have access to the transaction? }); } else { @@ -1135,17 +1164,17 @@ function Connection(args) { Easy, and very nifty. */ - conn.on('newListener', function (e, listener) { - if (e === 'String') { - // It's a string. - if (!(e in ['newListener'])) { - conn.notify(e, listener); - } - } - }); - conn.notify = function (name) { - - } + // conn.on('newListener', function (e, listener) { + // if (e === 'String') { + // // It's a string. + // if (!(e in ['newListener'])) { + // conn.notify(e, listener); + // } + // } + // }); + // conn.notify = function (name) { + // + // } } Connection.prototype = new process.EventEmitter(); @@ -1217,11 +1246,11 @@ function Transaction (connection /*, params */) { // We have an otherwise normal query. // This does not require a normal Sync message // or any other such magic-ery. - var q = new Query(sql, function (rs) { + var q = new Query(sql, function (err, rs) { if (exports.DEBUG > 3) { sys.debug("RS is:" + sys.inspect(rs)); } - callback(rs); + callback(err, rs); // callbackRun(); }); if (exports.DEBUG > 0) { From e60877253f1992d7a746607b1ac67d00b8de7181 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Wed, 27 Oct 2010 22:03:05 -0700 Subject: [PATCH 38/54] Adding error test runner, t/error.js Update the test runners and demo.js to use console.log instead of sys.puts. Updated the Query object to be compliant with multiple Queries in the same Transaction. Better support for system Transactions. --- demo.js | 49 ++++++++++++++------------- lib/postgres-pure.js | 78 ++++++++++++++++--------------------------- t/complex_prepared.js | 14 ++++---- t/error.js | 26 +++++++++++++++ t/query.js | 8 ++--- t/simple_prepared.js | 8 ++--- 6 files changed, 96 insertions(+), 87 deletions(-) create mode 100644 t/error.js diff --git a/demo.js b/demo.js index f811fd8..bdcb318 100644 --- a/demo.js +++ b/demo.js @@ -6,44 +6,47 @@ var db = new pg.connect("pgsql://test:12345@localhost:5432/returning_test"); db.query("SELECT 1::querytest;", function (error, rs, tx) { if (error) { console.log("Error!"); - sys.puts(error); - sys.puts(error.code); + console.log(error); + console.log(error.code); } else { - sys.puts(sys.inspect(rs)); + console.log(sys.inspect(rs)); } - // tx.query("SELECT 2::int as querytest2", function (rs) { - // sys.puts(sys.inspect(rs)); - // }); -}); - -db.prepare("INSERT INTO returning_test (val) VALUES (?) RETURNING id, val", function (sth) { - sth.execute("text value", function(e, rs) { - if (rs === undefined) { - console.log("No data."); - } - else { - console.log(sys.inspect(rs)); - } + tx.query("SELECT 2::int as querytest2", function (err, rs) { + console.log(sys.inspect(rs)); }); + tx.query("SELECT 3::qt" ,function (err, rs) { + console.log(sys.inspect(err)); + }) }); +// db.prepare("INSERT INTO returning_test (val) VALUES (?) RETURNING id, val", function (sth) { +// sth.execute("text value", function(e, rs) { +// if (rs === undefined) { +// console.log("No data."); +// } +// else { +// console.log(sys.inspect(rs)); +// } +// }); +// }); + // db.prepare().on("some_event"); // db.prepare("SELECT ?::int AS preparetest", function (sth, tx) { -// sth.execute(1, function (rs) { +// sth.execute(1, function (err, rs) { // sys.puts(sys.inspect(rs)); // }); -// sth.execute(2, function (rs) { +// sth.execute(2, function (err, rs) { // sys.puts(sys.inspect(rs)); // // }); -// // tx.prepare("SELECT ?::int AS preparetest2", function (sth) { -// // sth.execute(3, function (rs) { -// // sys.puts(sys.inspect(rs)); -// // }) ; -// // }); +// tx.prepare("SELECT ?::int AS preparetest2", function (sth) { +// sth.execute(3, function (err, rs) { +// sys.puts(sys.inspect(rs)); +// }) ; +// }); // }); // db.transaction(function (tx) { diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index d653121..0d6b71b 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -310,14 +310,14 @@ function message () { this._position = 0; // instance variable this.setMessages = function (msg) { if (msg instanceof Array) { - self.messages = msg; + this.messages = msg; } - self.length = self.messages.length; + this.length = this.messages.length; } this.addMessage = function (msg) { if (msg != null && msg != undefined) { - self.messages.push(msg); - self.length = self.messages.length; + this.messages.push(msg); + this.length = this.messages.length; } } this.next = function() { @@ -347,64 +347,57 @@ function message () { } return false; } - this.on("Error", function (err) { - if (errback !== null && errback !== undefined) { - errback(err); - } - else { - // We don't have an error handler? Odd.. - // Pass it up to our connection/transaction. - conn.emit("Error", err); - } - }); } message.prototype = new process.EventEmitter; // message.prototype.constructor = message; function Query(sql, callback) { + var self = this; message.call(this); this.sql = sql; - var q = this; - q.results = []; + this.results = []; var pos = 0; + this.callback = callback; /* Returns the next query object in this object buffer. This can be null. Should there be a Sync message here? */ - q.setMessages([ + this.setMessages([ { type: 'Query', - args: [sql] + args: [this.sql] }, // { // type: 'Flush', // args: [] // } ]); - q.on("newRow", function (row) { - q.results.push(row); + this.on("newRow", function (row) { + // sys.debug("Got row: " + sys.inspect(row)); + // sys.debug(sys.inspect(self.results)); + self.results.push(row); }); - q.on("Complete", function (data) { + this.on("Complete", function (data) { if (exports.DEBUG > 2) { - sys.debug("Callback: " + callback); + sys.debug("Callback: " + self.callback); } - callback(null, q.results); + self.callback(null, self.results); }); - q.toString = function () { return "Query: " + q.length}; - q.on("RowDescription", function (desc) { - q.row_description = desc; + this.toString = function () { return "Query: " + this.length}; + this.on("RowDescription", function (desc) { + self.row_description = desc; if (exports.DEBUG > 2) { sys.debug("Caught RowDescription message."); } }); - q.on("ErrorResponse", function (e) { - callback(e); + this.on("ErrorResponse", function (e) { + self.callback(e); }); - + // sys.debug(this.listeners("newRow")); } Query.prototype = new message; @@ -657,20 +650,6 @@ Prepared.prototype = new message(); Prepared.prototype.constructor = Prepared; -function Sync (callback) { - var q = this; - q.setMessages([ - { - type: "Sync", - args: null - }, - ]); - // -} - -Sync.prototype = new message; - - /* Initializes a connection to the database. DB connections are of the form: @@ -1064,10 +1043,10 @@ function Connection(args) { // No callback. args.push(callback); // re-add it. args.push(function (rs) { - // This will eventually be handled by an autocommit flag in - // the parameters object as passed to the connection. - - //tx.(); // implicit rollback. + /* + This becomes the implied callback. + For the moment, do nothing. + */ }); } tx.query.apply(tx, args); @@ -1228,7 +1207,7 @@ function Transaction (connection /*, params */) { /* sql, some_args, callback */ this.query = function () { var args = Array.prototype.slice.call(arguments); - if (exports.DEBUG>3) { + if (exports.DEBUG > 3) { sys.debug("Args are: " + args); } var sql = args.shift(); @@ -1246,6 +1225,7 @@ function Transaction (connection /*, params */) { // We have an otherwise normal query. // This does not require a normal Sync message // or any other such magic-ery. + var q = new Query(sql, function (err, rs) { if (exports.DEBUG > 3) { sys.debug("RS is:" + sys.inspect(rs)); @@ -1256,7 +1236,7 @@ function Transaction (connection /*, params */) { if (exports.DEBUG > 0) { sys.debug("New plain query created: "+q); } - thisp.push(q); + this.push(q); } } diff --git a/t/complex_prepared.js b/t/complex_prepared.js index 530065b..1b8f40f 100644 --- a/t/complex_prepared.js +++ b/t/complex_prepared.js @@ -4,20 +4,20 @@ pg.DEBUG=0; var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); -db.prepare("SELECT ?::int AS foobar", function (sth, tx) { - sth.execute(1, function (rs) { +db.prepare("SELECT ?::int AS foobar", function (err, sth, tx) { + sth.execute(1, function (err, rs) { console.log(eq(rs[0]['foobar'], 1)); - //sys.puts(sys.inspect(rs)); + //console.log(sys.inspect(rs)); }); - sth.execute(2, function (rs) { + sth.execute(2, function (err, rs) { console.log(eq(rs[0]['foobar'], 2)); - // sys.puts(sys.inspect(rs)); + // console.log(sys.inspect(rs)); }); tx.prepare("SELECT ?::int AS cheese", function (sth) { - sth.execute(3, function (rs) { + sth.execute(3, function (err, rs) { console.log(eq(rs[0]['cheese'], 3)); - // sys.puts(sys.inspect(rs)); + // console.log(sys.inspect(rs)); }); }); // db.close(); diff --git a/t/error.js b/t/error.js new file mode 100644 index 0000000..3aeabc8 --- /dev/null +++ b/t/error.js @@ -0,0 +1,26 @@ +/* Tests whether the error object is being correctly passed to the query +callback function */ +var sys = require("sys"); +var pg = require("../lib/postgres-pure"); +pg.DEBUG=0; + +var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); +db.query("SELECT 1::foobar;", function (err, rs, tx) { + console.log("error not null: " + not_eq(null, err)); + console.log("error code is 42704 (no such type foobar): " + eq("42704", err.code)); +}); +db.close(); + +function not_eq (l, r) { + if (l !== r) { + return "ok" + } + return "not ok\n " + l + " != " + r; +} + +function eq (l, r) { + if (l === r) { + return "ok" + } + return "not ok\n " + l + " != " + r; +} \ No newline at end of file diff --git a/t/query.js b/t/query.js index 0625288..9d20578 100644 --- a/t/query.js +++ b/t/query.js @@ -3,10 +3,10 @@ var pg = require("../lib/postgres-pure"); pg.DEBUG=0; var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); -db.query("SELECT 1::int as foobar;", function (rs, tx) { - sys.puts(sys.inspect(rs)); - tx.query("SELECT 2::int as foobartwo", function (rs) { - sys.puts(sys.inspect(rs)); +db.query("SELECT 1::int as foobar;", function (err, rs, tx) { + console.log(sys.inspect(rs)); + tx.query("SELECT 2::int as foobartwo", function (err, rs) { + console.log(sys.inspect(rs)); }); }); db.close(); \ No newline at end of file diff --git a/t/simple_prepared.js b/t/simple_prepared.js index ac3b814..fec67ab 100644 --- a/t/simple_prepared.js +++ b/t/simple_prepared.js @@ -4,11 +4,11 @@ pg.DEBUG=0; var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); db.prepare("SELECT ?::int AS foobar", function (sth, tx) { - sth.execute(1, function (rs) { - sys.puts(sys.inspect(rs)); + sth.execute(1, function (err, rs) { + console.log(sys.inspect(rs)); }); - sth.execute(2, function (rs) { - sys.puts(sys.inspect(rs)); + sth.execute(2, function (err, rs) { + console.log(sys.inspect(rs)); }); }); From 13041fb94cffea55a4e948d86365d2a86ae96d1b Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Wed, 27 Oct 2010 23:00:55 -0700 Subject: [PATCH 39/54] Adding tentative support for the transactional objects. --- demo.js | 82 +++++++++++++++++++++++++------------------- lib/postgres-pure.js | 37 ++++++++++++++++---- 2 files changed, 77 insertions(+), 42 deletions(-) diff --git a/demo.js b/demo.js index bdcb318..764d317 100644 --- a/demo.js +++ b/demo.js @@ -2,24 +2,24 @@ var sys = require("sys"); var pg = require("./lib/postgres-pure"); // pg.DEBUG=4; -var db = new pg.connect("pgsql://test:12345@localhost:5432/returning_test"); -db.query("SELECT 1::querytest;", function (error, rs, tx) { - if (error) { - console.log("Error!"); - console.log(error); - console.log(error.code); - } - else { - console.log(sys.inspect(rs)); - } - - tx.query("SELECT 2::int as querytest2", function (err, rs) { - console.log(sys.inspect(rs)); - }); - tx.query("SELECT 3::qt" ,function (err, rs) { - console.log(sys.inspect(err)); - }) -}); +var db = new pg.connect("pgsql://test:12345@localhost:5432/insert_test"); +// db.query("SELECT 1::querytest;", function (error, rs, tx) { +// if (error) { +// console.log("Error!"); +// console.log(error); +// console.log(error.code); +// } +// else { +// console.log(sys.inspect(rs)); +// } +// +// tx.query("SELECT 2::int as querytest2", function (err, rs) { +// console.log(sys.inspect(rs)); +// }); +// tx.query("SELECT 3::qt" ,function (err, rs) { +// console.log(sys.inspect(err)); +// }) +// }); // db.prepare("INSERT INTO returning_test (val) VALUES (?) RETURNING id, val", function (sth) { // sth.execute("text value", function(e, rs) { @@ -32,35 +32,47 @@ db.query("SELECT 1::querytest;", function (error, rs, tx) { // }); // }); -// db.prepare().on("some_event"); - // db.prepare("SELECT ?::int AS preparetest", function (sth, tx) { // sth.execute(1, function (err, rs) { -// sys.puts(sys.inspect(rs)); +// console.log(sys.inspect(rs)); // }); // sth.execute(2, function (err, rs) { -// sys.puts(sys.inspect(rs)); +// console.log(sys.inspect(rs)); // // }); // tx.prepare("SELECT ?::int AS preparetest2", function (sth) { // sth.execute(3, function (err, rs) { -// sys.puts(sys.inspect(rs)); +// console.log(sys.inspect(rs)); // }) ; // }); -// }); - -// db.transaction(function (tx) { -// // tx.begin(); -// tx.query("SELECT ?::int AS txtest1", 1, function (rs) { -// sys.puts(sys.inspect(rs)); -// }); -// tx.prepare("SELECT ?::int AS txtest2", function (sth) { -// sth.execute(2, function (rs) { -// sys.puts(sys.inspect(rs)); -// }); +// tx.prepare("SELECT ?::int AS preparetest2", function (sth) { +// sth.execute(4, function (err, rs) { +// console.log(sys.inspect(rs)); +// }) ; // }); -// // tx.commit(); // }); + +db.transaction(function (tx) { + tx.begin(); + // tx.query("INSERT INTO insert_test (val) VALUES (?) RETURNING id", "test value", function (err,rs) { + // console.log(sys.inspect(rs)); + // }); + // tx.prepare("INSERT INTO insert_test (val) VALUES (?) RETURNING id", function (sth) { + // sth.execute("twooooo", function (err, rs) { + // console.log(sys.inspect(rs)); + // }); + // }); + tx.query("SELECT 1::barrrrrr", function (err, rs) { + console.log(sys.inspect(err)); + }); + tx.query("SELECT 1::int as foobar", function (err, rs) { + console.log(sys.inspect(err)); + }); + tx.rollback(); + tx.query("SELECT 1::int as foobarbaz", function (err, rs) { + console.log(sys.inspect(rs)); + }); +}); db.close(); // db.prepare(query, function (sth, tx) { diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 0d6b71b..e0aa7b5 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -328,7 +328,7 @@ function message () { if (this.messages[this._position] !== null && this.messages[this._position] !== undefined) { this._position = this._position + 1; if (exports.DEBUG > 3){ - sys.debug("Returning: " + this.messages[this._position-1]); + sys.debug("Returning: " + sys.inspect(this.messages[this._position-1])); } return this.messages[this._position-1]; } @@ -608,8 +608,8 @@ function Prepared(sql, conn /*, use_named */) { eP.on("Complete", function () { if (exports.DEBUG > 0) { sys.debug("Results length " + eP.results.length); + sys.debug("noData is: "+eP.noData); } - sys.debug("noData is: "+eP.noData); if (eP.results.length > 0 && eP.bound) { if (exports.DEBUG > 0) { sys.debug("Execute Complete: Calling with results"); @@ -649,6 +649,18 @@ function Prepared(sql, conn /*, use_named */) { Prepared.prototype = new message(); Prepared.prototype.constructor = Prepared; +function Sync (callback) { + var q = this; + q.setMessages([ + { + type: "Sync", + args: null + }, + ]); +} + +Sync.prototype = new message; + /* Initializes a connection to the database. DB connections are of the form: @@ -1272,21 +1284,32 @@ function Transaction (connection /*, params */) { thisp.sync(); } + var tx_control_error = function (err, rs) { + if (null !== err) { + // if blue is not the sky, + thisp.sync(); // We need to clear the error. + // What else do we do as an error handler? + sys.debug("DB error: " + err); + } + // sys.debug("Callback called!"); + // do nothing else. We're good. + } + this.begin = function () { // Begins the transaction. We now lock the transaction to the wire. - thisp.push(thisp.query("BEGIN;")); // Don't need to watch for callbacks. + thisp.query("BEGIN", tx_control_error); } this.rollback = function () { // Rolls back the request, and does the connection release. + // this.sync(); + this.query("ROLLBACK", tx_control_error); - thisp.push(thisp.query("ROLLBACK;")); - thisp.sync(); } this.commit = function () { // Commits this block of stuff that's happened, via the SYNC message. // This will also cause a rollback if there's been errors. - thisp.push(thisp.query("COMMIT;")); - thisp.sync(); + // this.sync(); + this.query("COMMIT", tx_control_error); } this.sync = function () { From df1f350448fa7390546df8f84267c63d29da176c Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Wed, 27 Oct 2010 23:06:08 -0700 Subject: [PATCH 40/54] Transactions (basically) seem to work. No immediate side-effects to using transaction objects, or doing multiple queries in a given TX context. Added a basic test-case as t/transaction.js --- t/transaction.js | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 t/transaction.js diff --git a/t/transaction.js b/t/transaction.js new file mode 100644 index 0000000..8dd8541 --- /dev/null +++ b/t/transaction.js @@ -0,0 +1,25 @@ +var sys = require("sys"); +var pg = require("../lib/postgres-pure"); +pg.DEBUG=0; + +var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); +db.transaction(function (tx) { + tx.begin(); + tx.query("CREATE TABLE insert_test (id serial not null, val text)", function (err, rs) { + // Null query. + }); + tx.query("INSERT INTO insert_test (val) VALUES (?) RETURNING id", "test value", function (err,rs) { + console.log(sys.inspect(rs)); + }); + tx.prepare("INSERT INTO insert_test (val) VALUES (?) RETURNING id", function (sth) { + sth.execute("twooooo", function (err, rs) { + console.log(sys.inspect(rs)); + }); + }); + tx.rollback(); + tx.query("SELECT * FROM insert_test", function (err, rs) { + // Should error + console.log(sys.inspect(err)); + }); +}); +db.close(); \ No newline at end of file From cdbc49ee2b67093a7cf53cc895b3403505bd1031 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Wed, 8 Dec 2010 22:52:36 -0800 Subject: [PATCH 41/54] Adds better support for the driver under REPL conditions. Fixes a minor bug with the readme and pointing it to the correct URI. Adds the initial package.json for NPM support. --- lib/postgres-pure.js | 69 +++++++++++++++++++++++++++++++++----------- package.json | 10 +++++++ t/post_connect.js | 31 ++++++++++++++++++++ 3 files changed, 93 insertions(+), 17 deletions(-) create mode 100644 package.json create mode 100644 t/post_connect.js diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index e0aa7b5..bc39841 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -287,6 +287,10 @@ function parse_response(code, buffer) { type = "BindComplete"; args = [{}]; break; + case 'A': + type = "Notify"; + args = [{'pid': reader.int32(), 'name': reader.cstring(), 'payload': reader.cstring()}]; + break; } if (!type) { sys.debug("Unknown response " + code); @@ -700,6 +704,8 @@ function Connection(args) { results = []; queryEmpty = true; var wait = false; + + var notifications = new process.EventEmitter(); // Disable the idle timeout on the connection connection.setTimeout(0); @@ -869,12 +875,15 @@ function Connection(args) { }); events.on("canClose?", function () { - if (tx_queue.length == 0 && - current_tx.can_release() && - (current_query === null || - current_query.empty()) && - closeState && readyState + if ((tx_queue.length === 0 + && ( current_tx === null || current_tx.can_release() ) + && ( current_query === null || current_query.empty() ) ) // this must all be true. + && (closeState === true && readyState === true) ) { + if (exports.DEBUG > 1) { + sys.debug("Closing connection."); + } + closeState = true; connection.end(); } }); @@ -933,11 +942,14 @@ function Connection(args) { events.emit("nextTx"); } } + else if (tx_queue.length > 0) { + events.emit("nextTx"); + } else { - if (exports.DEBUG > 0) { - sys.debug("calling canClose? from nextQuery"); - } - events.emit("canClose?"); + // if (exports.DEBUG > 0) { + // sys.debug("calling canClose? from nextQuery"); + // } + // events.emit("canClose?"); } // } }); @@ -1035,6 +1047,13 @@ function Connection(args) { //readyState = true; }); + events.on("Notify", function (args) { + /* Name is the name of the notification + Payload is the string as sent from postgres. + */ + notifications.emit(args['name'], args['payload']); // Bubble through. + }); + conn.query = function query(/*query, args, callback */) { // Not sure I like this. @@ -1117,11 +1136,11 @@ function Connection(args) { connection.end(); } }; - // events.on("queryAdded", function () { - // if (readyState) { - // conn.emit("ReadyForQuery"); - // } - // }); + events.on("queryAdded", function () { + if (readyState) { + events.emit("nextMessage"); + } + }); // Pushes a new transaction into the transaction pool, assuming @@ -1163,9 +1182,18 @@ function Connection(args) { // } // } // }); - // conn.notify = function (name) { - // - // } + conn.notify = function (name, callback) { + // Sets up a listener for the NOTIFY event from the database. + // Works by queuing up a LISTEN event in the DB (if it's not already + // been registered on this connection), and adding the event (with + // callback) to the event. + + var tx = conn.tx(); // Internal abstractions are sometimes silly. + tx.push(new Listener(name)); + tx_queue.push(tx); + conn.notifications.on(name, callback ); + events.emit("queryAdded"); + } } Connection.prototype = new process.EventEmitter(); @@ -1355,8 +1383,15 @@ function Transaction (connection /*, params */) { Transaction.prototype = new process.EventEmitter(); // This will eventually be the return object +/* +Should expose the same API as the Connection object, just hold multiple copies +of connections open for better performance. +Woo. +Notifications get written across all possible connections (just in case) +*/ function connectionManager (dsn /*, connections=1 */) { // var conn = new Connection(dsn); + } exports.connect = Connection; \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..d291154 --- /dev/null +++ b/package.json @@ -0,0 +1,10 @@ +{"name": "postgres", + "version": "0.1.0RC1", + "description": "Pure-JavaScript implementation of the Postgres Protocol", + "homepage": "https://public.commandprompt.com/projects/postgres-js/", + "author": { + "name": "Aurynn Shaw", + "email": "ashaw@commandprompt.com", + }, + "main": "lib/postgres-pure", +} \ No newline at end of file diff --git a/t/post_connect.js b/t/post_connect.js new file mode 100644 index 0000000..c448988 --- /dev/null +++ b/t/post_connect.js @@ -0,0 +1,31 @@ +/* Tests whether the error object is being correctly passed to the query +callback function */ +var sys = require("sys"); +var pg = require("../lib/postgres-pure"); +pg.DEBUG=0; + +var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); +db.on("connection", function () { + db.query("SELECT 1::int", function (err, rs, tx) { + // verify that the error is null, and everything is okay. + console.log(eq(err, null)); + // Test if the returned value is what we think it is. + console.log(eq(rs[0]['int4'], 1)); + }); + db.close(); +}); + + +function not_eq (l, r) { + if (l !== r) { + return "ok" + } + return "not ok " + l + " != " + r; +} + +function eq (l, r) { + if (l === r) { + return "ok" + } + return "not ok " + l + " != " + r; +} \ No newline at end of file From 4b2d995c6d2020d51968c9ceea4a8eb07197827b Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Thu, 9 Dec 2010 22:26:50 -0800 Subject: [PATCH 42/54] Added notify support to the driver. Added t/notify.js for a quick test case for notify. --- lib/postgres-pure.js | 24 ++++++++---------------- t/notify.js | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 16 deletions(-) create mode 100644 t/notify.js diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index bc39841..ac97511 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -653,18 +653,6 @@ function Prepared(sql, conn /*, use_named */) { Prepared.prototype = new message(); Prepared.prototype.constructor = Prepared; -function Sync (callback) { - var q = this; - q.setMessages([ - { - type: "Sync", - args: null - }, - ]); -} - -Sync.prototype = new message; - /* Initializes a connection to the database. DB connections are of the form: @@ -1188,10 +1176,14 @@ function Connection(args) { // been registered on this connection), and adding the event (with // callback) to the event. - var tx = conn.tx(); // Internal abstractions are sometimes silly. - tx.push(new Listener(name)); - tx_queue.push(tx); - conn.notifications.on(name, callback ); + this.query("LISTEN "+ name, function (err,rs) { + /* Doesn't need to do anything */ + if (err !== null) { + callback(err); + } + }); + + notifications.on( name, function (payload) { callback(null, payload) }); events.emit("queryAdded"); } } diff --git a/t/notify.js b/t/notify.js new file mode 100644 index 0000000..747526c --- /dev/null +++ b/t/notify.js @@ -0,0 +1,32 @@ +/* Tests whether the error object is being correctly passed to the query +callback function */ +var sys = require("sys"); +var pg = require("../lib/postgres-pure"); +pg.DEBUG=0; + +var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); +var db2 = new pg.connect("pgsql://test:12345@localhost:5432/template1"); + +db.notify("test", function (err, payload) { + console.log(eq(err, null)); +}); +db2.on("connection", function () { + db2.query("NOTIFY test", function (err,rs) {}); // Null + db2.close(); +}); +db.close(); + + +function not_eq (l, r) { + if (l !== r) { + return "ok" + } + return "not ok " + l + " != " + r; +} + +function eq (l, r) { + if (l === r) { + return "ok" + } + return "not ok " + l + " != " + r; +} \ No newline at end of file From 3f18b7b3d230027e016716345a2a83f216f4f023 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Thu, 9 Dec 2010 22:56:11 -0800 Subject: [PATCH 43/54] Fixing the readme. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 230ddf1..ef8569b 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ All code is available under the terms of the MIT license, unless otherwise noted (c) 2010, Tim Caswell, Aurynn Shaw. -Bugs can be reported @ https://public.commandprompt.com/projects/postgres-js +Bugs can be reported @ https://public.commandprompt.com/projects/postgresjs ## Example use From 2d0ee107534ba5413ae0380648535320d62c6f9d Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Tue, 28 Dec 2010 13:29:24 -0700 Subject: [PATCH 44/54] Modification to make postgres.js not advance to the next TX block if it's still in a TX. Updated the package.json Modifying the complex prepared test. --- lib/postgres-pure.js | 38 +++++++++++++++++++++----------------- package.json | 2 +- t/complex_prepared.js | 2 +- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index ac97511..0e1c851 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -692,6 +692,7 @@ function Connection(args) { results = []; queryEmpty = true; var wait = false; + var txState = null; var notifications = new process.EventEmitter(); @@ -834,6 +835,7 @@ function Connection(args) { if (exports.DEBUG > 0) { sys.debug("In RFQ"); } + txState = state; if (!started) { started = true; conn.emit('connection'); @@ -946,30 +948,32 @@ function Connection(args) { if (exports.DEBUG > 0) { sys.debug("got nextTx"); } - if (readyState) { - if (tx_queue.length > 0) { - if (exports.DEBUG > 1) { - sys.debug("nextTx: tx_queue length is: " + tx_queue.length); - } - current_tx = tx_queue.shift(); - if (current_tx !== null && current_tx !== undefined) { - if (exports.DEBUG>2) { - sys.debug("nextTx: current_tx is " + sys.inspect(current_tx)); + if (txState == 'I') { + if (readyState) { + if (tx_queue.length > 0) { + if (exports.DEBUG > 1) { + sys.debug("nextTx: tx_queue length is: " + tx_queue.length); + } + current_tx = tx_queue.shift(); + if (current_tx !== null && current_tx !== undefined) { + if (exports.DEBUG>2) { + sys.debug("nextTx: current_tx is " + sys.inspect(current_tx)); + } + events.emit("nextQuery"); + } + else { + if (exports.DEBUG > 0) { + sys.debug("next TX is null."); + } } - events.emit("nextQuery"); } else { if (exports.DEBUG > 0) { - sys.debug("next TX is null."); + sys.debug("calling canClose? from nextTx"); } + events.emit("canClose?"); } } - else { - if (exports.DEBUG > 0) { - sys.debug("calling canClose? from nextTx"); - } - events.emit("canClose?"); - } } }); diff --git a/package.json b/package.json index d291154..b6566a8 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ {"name": "postgres", - "version": "0.1.0RC1", + "version": "0.1.0RC2", "description": "Pure-JavaScript implementation of the Postgres Protocol", "homepage": "https://public.commandprompt.com/projects/postgres-js/", "author": { diff --git a/t/complex_prepared.js b/t/complex_prepared.js index 1b8f40f..e9066d0 100644 --- a/t/complex_prepared.js +++ b/t/complex_prepared.js @@ -4,7 +4,7 @@ pg.DEBUG=0; var db = new pg.connect("pgsql://test:12345@localhost:5432/template1"); -db.prepare("SELECT ?::int AS foobar", function (err, sth, tx) { +db.prepare("SELECT ?::int AS foobar", function (sth, tx) { sth.execute(1, function (err, rs) { console.log(eq(rs[0]['foobar'], 1)); //console.log(sys.inspect(rs)); From 04ba97ff3f2c17bc046d8a0407d0cacfdf8c9802 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Fri, 31 Dec 2010 20:02:19 -0700 Subject: [PATCH 45/54] Zero-length resultsets are no longer an error. --- lib/postgres-pure.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 0e1c851..01ea72d 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -628,7 +628,7 @@ function Prepared(sql, conn /*, use_named */) { callback(); } else { - eP.emit("error", "Complete reached without records or NoData state."); + callback(null, []); } }); eP.on("BindComplete", function () { From fdd01d20d49952bdb43b36c777f9ca740d60c290 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Sat, 22 Jan 2011 09:35:15 -0700 Subject: [PATCH 46/54] Added support for date objects to be converted to and from postgres timestamptz. --- lib/postgres-pure.js | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 01ea72d..1f9aa6f 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -62,8 +62,13 @@ var formatter = { case "number": case "boolean": case "object": - builder.push.int16(1); // binary. This will need to be - // careful, as Date will be an object. + if (args[i] instanceof Date) { + builder.push.int16(0); // Dates are Strings. + } + else { + builder.push.int16(1); // binary. This will need to be + // careful, as Date will be an object. + } break; case "string": // Any other type. builder.push.int16(0); @@ -87,8 +92,22 @@ var formatter = { .push.string(args[i] ? 1 : 0); break; case "object": + if (args[i] instanceof Date) { + + // This uses the UTC string, and tells PG to be in the UTC + // timezone, for ease of use. + // And it kind of requires that all fields in PG be timestamptz... + var l = args[i].toLocaleString(); + var a = l.split(" "); + var tz = a[a.length-2]; + a.length = a.length - 2; // Truncate the array + var v = a.join(" ") + "-" + tz.split("-")[1].slice(0, 2); + builder.push.int32(v.length).push.string(v); + break; + } if (args[i] === null) { builder.push.int32(-1); + break; } }; } @@ -996,8 +1015,7 @@ function Connection(args) { description = current_query.row_description[i]; value = data[i]; if (value !== null) { - // Type OIDs are stable. - // TODO: Investigate javascript Date objects. + // Type OIDs are stable, more or less. They're unlikely to change. // see pg_type.h for the defined values. switch (description.type_id) { case 16: // bool @@ -1008,6 +1026,21 @@ function Connection(args) { case 23: // int4 value = parseInt(value, 10); break; + case 1082: // Date + value = new Date(value); + case 1114: // Timestamp, no timezone + value = new Date(value); + case 1184: // Timestamp, with timezone + if (value[value.length-1].toLowerCase() == 'z') { + // It's in UTC time + // So, we add the appropriate modifiers. + value = new Date(value.slice(0, value.length-1) + " GMT+0000" ); + } + else { + var tz = value.slice(value.length-3, value.length); // last three. + var orig = value; + value = new Date(value.slice(0, value.length-3) + " GMT"+tz+"00"); + } } } row[description.field] = value; From 0c5293815a91de3755f201abf56ab02458a55ccc Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Tue, 1 Feb 2011 15:00:45 -0700 Subject: [PATCH 47/54] Tweaks to the NULL and date handling in postgres.js --- benchmark.js | 26 ++++++++++++++ demo.js | 86 +++++++++++++++++++++++++------------------- lib/postgres-pure.js | 13 ++++--- package.json | 5 ++- 4 files changed, 87 insertions(+), 43 deletions(-) create mode 100644 benchmark.js diff --git a/benchmark.js b/benchmark.js new file mode 100644 index 0000000..337bcd7 --- /dev/null +++ b/benchmark.js @@ -0,0 +1,26 @@ +var sys = require("sys"); + +var iterations = process.argv[2]; + +var pg = require("./lib/postgres-pure"); +var db = new pg.connect("pgsql://test:12345@localhost:5432/insert_test"); +db.transaction(function (tx) { + tx.query("CREATE TABLE insert_test (id serial not null, val text)", function (err, rs) {}); + tx.begin(); + tx.prepare("INSERT INTO insert_test (val) VALUES (?::text) RETURNING id, val", function (sth) { + var i = 0; + for (i = 0; i <= iterations; i++) { + sth.execute(""+i, function (err, rs) { + // This is the point where we have executed. + console.log(rs[0]); + }) + } + }); + tx.query("SELECT COUNT(*) FROM insert_test", function (err, rs) { + console.log(sys.inspect(rs)); + }); + tx.rollback(); + tx.query("DROP TABLE insert_test", function () {}); +}); +console.log("Done"); +db.close(); \ No newline at end of file diff --git a/demo.js b/demo.js index 764d317..36e2c69 100644 --- a/demo.js +++ b/demo.js @@ -3,6 +3,18 @@ var pg = require("./lib/postgres-pure"); // pg.DEBUG=4; var db = new pg.connect("pgsql://test:12345@localhost:5432/insert_test"); + +// db.query("SELECT 1::errortest;", function (error, rs, tx) { +// if (error) { +// console.log("Error!"); +// console.log(error); +// console.log(error.code); +// } +// else { +// console.log(sys.inspect(rs)); +// } +// }); + // db.query("SELECT 1::querytest;", function (error, rs, tx) { // if (error) { // console.log("Error!"); @@ -32,47 +44,47 @@ var db = new pg.connect("pgsql://test:12345@localhost:5432/insert_test"); // }); // }); -// db.prepare("SELECT ?::int AS preparetest", function (sth, tx) { -// sth.execute(1, function (err, rs) { -// console.log(sys.inspect(rs)); -// }); -// sth.execute(2, function (err, rs) { -// console.log(sys.inspect(rs)); -// -// }); -// tx.prepare("SELECT ?::int AS preparetest2", function (sth) { -// sth.execute(3, function (err, rs) { -// console.log(sys.inspect(rs)); -// }) ; -// }); -// tx.prepare("SELECT ?::int AS preparetest2", function (sth) { -// sth.execute(4, function (err, rs) { -// console.log(sys.inspect(rs)); -// }) ; -// }); -// }); - -db.transaction(function (tx) { - tx.begin(); - // tx.query("INSERT INTO insert_test (val) VALUES (?) RETURNING id", "test value", function (err,rs) { - // console.log(sys.inspect(rs)); - // }); - // tx.prepare("INSERT INTO insert_test (val) VALUES (?) RETURNING id", function (sth) { - // sth.execute("twooooo", function (err, rs) { - // console.log(sys.inspect(rs)); - // }); - // }); - tx.query("SELECT 1::barrrrrr", function (err, rs) { - console.log(sys.inspect(err)); - }); - tx.query("SELECT 1::int as foobar", function (err, rs) { - console.log(sys.inspect(err)); +db.prepare("SELECT ?::int AS preparetest", function (sth, tx) { + sth.execute(1, function (err, rs) { + console.log(sys.inspect(rs)); }); - tx.rollback(); - tx.query("SELECT 1::int as foobarbaz", function (err, rs) { + sth.execute(2, function (err, rs) { console.log(sys.inspect(rs)); + + }); + tx.prepare("SELECT ?::int AS preparetest2", function (sth) { + sth.execute(3, function (err, rs) { + console.log(sys.inspect(rs)); + }) ; + }); + tx.prepare("SELECT ?::int AS preparetest2", function (sth) { + sth.execute(4, function (err, rs) { + console.log(sys.inspect(rs)); + }) ; }); }); + +// db.transaction(function (tx) { +// tx.begin(); +// // tx.query("INSERT INTO insert_test (val) VALUES (?) RETURNING id", "test value", function (err,rs) { +// // console.log(sys.inspect(rs)); +// // }); +// // tx.prepare("INSERT INTO insert_test (val) VALUES (?) RETURNING id", function (sth) { +// // sth.execute("twooooo", function (err, rs) { +// // console.log(sys.inspect(rs)); +// // }); +// // }); +// tx.query("SELECT 1::barrrrrr", function (err, rs) { +// console.log(sys.inspect(err)); +// }); +// tx.query("SELECT 1::int as foobar", function (err, rs) { +// console.log(sys.inspect(err)); +// }); +// tx.rollback(); +// tx.query("SELECT 1::int as foobarbaz", function (err, rs) { +// console.log(sys.inspect(rs)); +// }); +// }); db.close(); // db.prepare(query, function (sth, tx) { diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 1f9aa6f..0086c48 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -55,8 +55,9 @@ var formatter = { var builder = (encoder('B')) .push.cstring(portal) .push.cstring(prepared_name) - .push.int16(args.length); // declare our format codes as expected. - + // .push.int16(args.length); // declare our format codes as expected. + .push.int16(0); // Text + /* for (var i = 0; i < args.length; i++) { switch (typeof args[i]) { case "number": @@ -75,13 +76,14 @@ var formatter = { break; } } + */ builder.push.int16(args.length); for (var i = 0; i < args.length; i++) { switch (typeof args[i]) { case "number": - builder.push.int32(4) // 4 bytes. int32. - .push.int32(args[i]); + builder.push.int32(args[i].toString().length) // 4 bytes. int32. + .push.string(args[i].toString()); break; case "string": builder.push.int32(args[i].length) @@ -105,7 +107,8 @@ var formatter = { builder.push.int32(v.length).push.string(v); break; } - if (args[i] === null) { + if (args[i] === null || args[i] === undefined) { + // this isn't right... builder.push.int32(-1); break; } diff --git a/package.json b/package.json index b6566a8..251d3f7 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ {"name": "postgres", - "version": "0.1.0RC2", + "version": "0.1.0", "description": "Pure-JavaScript implementation of the Postgres Protocol", "homepage": "https://public.commandprompt.com/projects/postgres-js/", "author": { @@ -7,4 +7,7 @@ "email": "ashaw@commandprompt.com", }, "main": "lib/postgres-pure", + "dependencies" : + { "strtok" : ">=0.1.1" + } } \ No newline at end of file From 46e44561afee7f71f30ae1a776b05ad4371f88e2 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Tue, 1 Feb 2011 15:12:38 -0700 Subject: [PATCH 48/54] Going to 0.2 development. 0.2 will replace the buffer implementation with strtok and switch to crypto for md5 handling. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 251d3f7..ec1a0d1 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ {"name": "postgres", - "version": "0.1.0", + "version": "0.2-dev", "description": "Pure-JavaScript implementation of the Postgres Protocol", "homepage": "https://public.commandprompt.com/projects/postgres-js/", "author": { From dd3ea80ee3f3c8e6fd80c711b8b352f6f6d32a41 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Tue, 1 Feb 2011 23:48:33 -0700 Subject: [PATCH 49/54] Fixed timestamptz handling again. Still needs to explicitly set the DATESTYLE on connect. --- lib/postgres-pure.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 0086c48..1d004dc 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -1034,15 +1034,23 @@ function Connection(args) { case 1114: // Timestamp, no timezone value = new Date(value); case 1184: // Timestamp, with timezone + // Initial value: + // "2011-02-01 21:00:52.353444-07" + // Needs to become: + // "2011-02-01 21:00:52.353 GMT-0700" if (value[value.length-1].toLowerCase() == 'z') { // It's in UTC time // So, we add the appropriate modifiers. - value = new Date(value.slice(0, value.length-1) + " GMT+0000" ); + var tz = value.slice(0, value.length-1); + tz = tz.slice(0, tz.length-4); + value = new Date(tz + " GMT+0000" ); + console.log(value); } else { var tz = value.slice(value.length-3, value.length); // last three. var orig = value; - value = new Date(value.slice(0, value.length-3) + " GMT"+tz+"00"); + orig = orig.slice(0, orig.length-7); + value = new Date(orig + " GMT"+tz+"00"); } } } From b3bc897615ad6bfe350558375f5971909c42bc38 Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Fri, 11 Feb 2011 17:49:44 -0700 Subject: [PATCH 50/54] Switching from md5.js to using Node's core crypto library. --- lib/md5.js | 381 ------------------------------------------- lib/postgres-pure.js | 13 +- package.json | 4 +- t/insert.js | 26 +++ 4 files changed, 34 insertions(+), 390 deletions(-) delete mode 100644 lib/md5.js create mode 100644 t/insert.js diff --git a/lib/md5.js b/lib/md5.js deleted file mode 100644 index 1e8f3b6..0000000 --- a/lib/md5.js +++ /dev/null @@ -1,381 +0,0 @@ -/* - * A JavaScript implementation of the RSA Data Security, Inc. MD5 Message - * Digest Algorithm, as defined in RFC 1321. - * Version 2.2 Copyright (C) Paul Johnston 1999 - 2009 - * Other contributors: Greg Holt, Andrew Kepert, Ydnar, Lostinet - * Distributed under the BSD License - * See http://pajhome.org.uk/crypt/md5 for more info. - */ - -/* - * Configurable variables. You may need to tweak these to be compatible with - * the server-side, but the defaults work in most cases. - */ -var hexcase = 0; /* hex output format. 0 - lowercase; 1 - uppercase */ -var b64pad = ""; /* base-64 pad character. "=" for strict RFC compliance */ - -/* - * These are the functions you'll usually want to call - * They take string arguments and return either hex or base-64 encoded strings - */ -function hex_md5(s) { return rstr2hex(rstr_md5(str2rstr_utf8(s))); } -function b64_md5(s) { return rstr2b64(rstr_md5(str2rstr_utf8(s))); } -function any_md5(s, e) { return rstr2any(rstr_md5(str2rstr_utf8(s)), e); } -function hex_hmac_md5(k, d) - { return rstr2hex(rstr_hmac_md5(str2rstr_utf8(k), str2rstr_utf8(d))); } -function b64_hmac_md5(k, d) - { return rstr2b64(rstr_hmac_md5(str2rstr_utf8(k), str2rstr_utf8(d))); } -function any_hmac_md5(k, d, e) - { return rstr2any(rstr_hmac_md5(str2rstr_utf8(k), str2rstr_utf8(d)), e); } - -module.exports = function md5(s) { return rstr2hex(rstr_md5(s)); } - -/* - * Perform a simple self-test to see if the VM is working - */ -function md5_vm_test() -{ - return hex_md5("abc").toLowerCase() == "900150983cd24fb0d6963f7d28e17f72"; -} - -/* - * Calculate the MD5 of a raw string - */ -function rstr_md5(s) -{ - return binl2rstr(binl_md5(rstr2binl(s), s.length * 8)); -} - -/* - * Calculate the HMAC-MD5, of a key and some data (raw strings) - */ -function rstr_hmac_md5(key, data) -{ - var bkey = rstr2binl(key); - if(bkey.length > 16) bkey = binl_md5(bkey, key.length * 8); - - var ipad = Array(16), opad = Array(16); - for(var i = 0; i < 16; i++) - { - ipad[i] = bkey[i] ^ 0x36363636; - opad[i] = bkey[i] ^ 0x5C5C5C5C; - } - - var hash = binl_md5(ipad.concat(rstr2binl(data)), 512 + data.length * 8); - return binl2rstr(binl_md5(opad.concat(hash), 512 + 128)); -} - -/* - * Convert a raw string to a hex string - */ -function rstr2hex(input) -{ - try { hexcase } catch(e) { hexcase=0; } - var hex_tab = hexcase ? "0123456789ABCDEF" : "0123456789abcdef"; - var output = ""; - var x; - for(var i = 0; i < input.length; i++) - { - x = input.charCodeAt(i); - output += hex_tab.charAt((x >>> 4) & 0x0F) - + hex_tab.charAt( x & 0x0F); - } - return output; -} - -/* - * Convert a raw string to a base-64 string - */ -function rstr2b64(input) -{ - try { b64pad } catch(e) { b64pad=''; } - var tab = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - var output = ""; - var len = input.length; - for(var i = 0; i < len; i += 3) - { - var triplet = (input.charCodeAt(i) << 16) - | (i + 1 < len ? input.charCodeAt(i+1) << 8 : 0) - | (i + 2 < len ? input.charCodeAt(i+2) : 0); - for(var j = 0; j < 4; j++) - { - if(i * 8 + j * 6 > input.length * 8) output += b64pad; - else output += tab.charAt((triplet >>> 6*(3-j)) & 0x3F); - } - } - return output; -} - -/* - * Convert a raw string to an arbitrary string encoding - */ -function rstr2any(input, encoding) -{ - var divisor = encoding.length; - var i, j, q, x, quotient; - - /* Convert to an array of 16-bit big-endian values, forming the dividend */ - var dividend = Array(Math.ceil(input.length / 2)); - for(i = 0; i < dividend.length; i++) - { - dividend[i] = (input.charCodeAt(i * 2) << 8) | input.charCodeAt(i * 2 + 1); - } - - /* - * Repeatedly perform a long division. The binary array forms the dividend, - * the length of the encoding is the divisor. Once computed, the quotient - * forms the dividend for the next step. All remainders are stored for later - * use. - */ - var full_length = Math.ceil(input.length * 8 / - (Math.log(encoding.length) / Math.log(2))); - var remainders = Array(full_length); - for(j = 0; j < full_length; j++) - { - quotient = Array(); - x = 0; - for(i = 0; i < dividend.length; i++) - { - x = (x << 16) + dividend[i]; - q = Math.floor(x / divisor); - x -= q * divisor; - if(quotient.length > 0 || q > 0) - quotient[quotient.length] = q; - } - remainders[j] = x; - dividend = quotient; - } - - /* Convert the remainders to the output string */ - var output = ""; - for(i = remainders.length - 1; i >= 0; i--) - output += encoding.charAt(remainders[i]); - - return output; -} - -/* - * Encode a string as utf-8. - * For efficiency, this assumes the input is valid utf-16. - */ -function str2rstr_utf8(input) -{ - var output = ""; - var i = -1; - var x, y; - - while(++i < input.length) - { - /* Decode utf-16 surrogate pairs */ - x = input.charCodeAt(i); - y = i + 1 < input.length ? input.charCodeAt(i + 1) : 0; - if(0xD800 <= x && x <= 0xDBFF && 0xDC00 <= y && y <= 0xDFFF) - { - x = 0x10000 + ((x & 0x03FF) << 10) + (y & 0x03FF); - i++; - } - - /* Encode output as utf-8 */ - if(x <= 0x7F) - output += String.fromCharCode(x); - else if(x <= 0x7FF) - output += String.fromCharCode(0xC0 | ((x >>> 6 ) & 0x1F), - 0x80 | ( x & 0x3F)); - else if(x <= 0xFFFF) - output += String.fromCharCode(0xE0 | ((x >>> 12) & 0x0F), - 0x80 | ((x >>> 6 ) & 0x3F), - 0x80 | ( x & 0x3F)); - else if(x <= 0x1FFFFF) - output += String.fromCharCode(0xF0 | ((x >>> 18) & 0x07), - 0x80 | ((x >>> 12) & 0x3F), - 0x80 | ((x >>> 6 ) & 0x3F), - 0x80 | ( x & 0x3F)); - } - return output; -} - -/* - * Encode a string as utf-16 - */ -function str2rstr_utf16le(input) -{ - var output = ""; - for(var i = 0; i < input.length; i++) - output += String.fromCharCode( input.charCodeAt(i) & 0xFF, - (input.charCodeAt(i) >>> 8) & 0xFF); - return output; -} - -function str2rstr_utf16be(input) -{ - var output = ""; - for(var i = 0; i < input.length; i++) - output += String.fromCharCode((input.charCodeAt(i) >>> 8) & 0xFF, - input.charCodeAt(i) & 0xFF); - return output; -} - -/* - * Convert a raw string to an array of little-endian words - * Characters >255 have their high-byte silently ignored. - */ -function rstr2binl(input) -{ - var output = Array(input.length >> 2); - for(var i = 0; i < output.length; i++) - output[i] = 0; - for(var i = 0; i < input.length * 8; i += 8) - output[i>>5] |= (input.charCodeAt(i / 8) & 0xFF) << (i%32); - return output; -} - -/* - * Convert an array of little-endian words to a string - */ -function binl2rstr(input) -{ - var output = ""; - for(var i = 0; i < input.length * 32; i += 8) - output += String.fromCharCode((input[i>>5] >>> (i % 32)) & 0xFF); - return output; -} - -/* - * Calculate the MD5 of an array of little-endian words, and a bit length. - */ -function binl_md5(x, len) -{ - /* append padding */ - x[len >> 5] |= 0x80 << ((len) % 32); - x[(((len + 64) >>> 9) << 4) + 14] = len; - - var a = 1732584193; - var b = -271733879; - var c = -1732584194; - var d = 271733878; - - for(var i = 0; i < x.length; i += 16) - { - var olda = a; - var oldb = b; - var oldc = c; - var oldd = d; - - a = md5_ff(a, b, c, d, x[i+ 0], 7 , -680876936); - d = md5_ff(d, a, b, c, x[i+ 1], 12, -389564586); - c = md5_ff(c, d, a, b, x[i+ 2], 17, 606105819); - b = md5_ff(b, c, d, a, x[i+ 3], 22, -1044525330); - a = md5_ff(a, b, c, d, x[i+ 4], 7 , -176418897); - d = md5_ff(d, a, b, c, x[i+ 5], 12, 1200080426); - c = md5_ff(c, d, a, b, x[i+ 6], 17, -1473231341); - b = md5_ff(b, c, d, a, x[i+ 7], 22, -45705983); - a = md5_ff(a, b, c, d, x[i+ 8], 7 , 1770035416); - d = md5_ff(d, a, b, c, x[i+ 9], 12, -1958414417); - c = md5_ff(c, d, a, b, x[i+10], 17, -42063); - b = md5_ff(b, c, d, a, x[i+11], 22, -1990404162); - a = md5_ff(a, b, c, d, x[i+12], 7 , 1804603682); - d = md5_ff(d, a, b, c, x[i+13], 12, -40341101); - c = md5_ff(c, d, a, b, x[i+14], 17, -1502002290); - b = md5_ff(b, c, d, a, x[i+15], 22, 1236535329); - - a = md5_gg(a, b, c, d, x[i+ 1], 5 , -165796510); - d = md5_gg(d, a, b, c, x[i+ 6], 9 , -1069501632); - c = md5_gg(c, d, a, b, x[i+11], 14, 643717713); - b = md5_gg(b, c, d, a, x[i+ 0], 20, -373897302); - a = md5_gg(a, b, c, d, x[i+ 5], 5 , -701558691); - d = md5_gg(d, a, b, c, x[i+10], 9 , 38016083); - c = md5_gg(c, d, a, b, x[i+15], 14, -660478335); - b = md5_gg(b, c, d, a, x[i+ 4], 20, -405537848); - a = md5_gg(a, b, c, d, x[i+ 9], 5 , 568446438); - d = md5_gg(d, a, b, c, x[i+14], 9 , -1019803690); - c = md5_gg(c, d, a, b, x[i+ 3], 14, -187363961); - b = md5_gg(b, c, d, a, x[i+ 8], 20, 1163531501); - a = md5_gg(a, b, c, d, x[i+13], 5 , -1444681467); - d = md5_gg(d, a, b, c, x[i+ 2], 9 , -51403784); - c = md5_gg(c, d, a, b, x[i+ 7], 14, 1735328473); - b = md5_gg(b, c, d, a, x[i+12], 20, -1926607734); - - a = md5_hh(a, b, c, d, x[i+ 5], 4 , -378558); - d = md5_hh(d, a, b, c, x[i+ 8], 11, -2022574463); - c = md5_hh(c, d, a, b, x[i+11], 16, 1839030562); - b = md5_hh(b, c, d, a, x[i+14], 23, -35309556); - a = md5_hh(a, b, c, d, x[i+ 1], 4 , -1530992060); - d = md5_hh(d, a, b, c, x[i+ 4], 11, 1272893353); - c = md5_hh(c, d, a, b, x[i+ 7], 16, -155497632); - b = md5_hh(b, c, d, a, x[i+10], 23, -1094730640); - a = md5_hh(a, b, c, d, x[i+13], 4 , 681279174); - d = md5_hh(d, a, b, c, x[i+ 0], 11, -358537222); - c = md5_hh(c, d, a, b, x[i+ 3], 16, -722521979); - b = md5_hh(b, c, d, a, x[i+ 6], 23, 76029189); - a = md5_hh(a, b, c, d, x[i+ 9], 4 , -640364487); - d = md5_hh(d, a, b, c, x[i+12], 11, -421815835); - c = md5_hh(c, d, a, b, x[i+15], 16, 530742520); - b = md5_hh(b, c, d, a, x[i+ 2], 23, -995338651); - - a = md5_ii(a, b, c, d, x[i+ 0], 6 , -198630844); - d = md5_ii(d, a, b, c, x[i+ 7], 10, 1126891415); - c = md5_ii(c, d, a, b, x[i+14], 15, -1416354905); - b = md5_ii(b, c, d, a, x[i+ 5], 21, -57434055); - a = md5_ii(a, b, c, d, x[i+12], 6 , 1700485571); - d = md5_ii(d, a, b, c, x[i+ 3], 10, -1894986606); - c = md5_ii(c, d, a, b, x[i+10], 15, -1051523); - b = md5_ii(b, c, d, a, x[i+ 1], 21, -2054922799); - a = md5_ii(a, b, c, d, x[i+ 8], 6 , 1873313359); - d = md5_ii(d, a, b, c, x[i+15], 10, -30611744); - c = md5_ii(c, d, a, b, x[i+ 6], 15, -1560198380); - b = md5_ii(b, c, d, a, x[i+13], 21, 1309151649); - a = md5_ii(a, b, c, d, x[i+ 4], 6 , -145523070); - d = md5_ii(d, a, b, c, x[i+11], 10, -1120210379); - c = md5_ii(c, d, a, b, x[i+ 2], 15, 718787259); - b = md5_ii(b, c, d, a, x[i+ 9], 21, -343485551); - - a = safe_add(a, olda); - b = safe_add(b, oldb); - c = safe_add(c, oldc); - d = safe_add(d, oldd); - } - return Array(a, b, c, d); -} - -/* - * These functions implement the four basic operations the algorithm uses. - */ -function md5_cmn(q, a, b, x, s, t) -{ - return safe_add(bit_rol(safe_add(safe_add(a, q), safe_add(x, t)), s),b); -} -function md5_ff(a, b, c, d, x, s, t) -{ - return md5_cmn((b & c) | ((~b) & d), a, b, x, s, t); -} -function md5_gg(a, b, c, d, x, s, t) -{ - return md5_cmn((b & d) | (c & (~d)), a, b, x, s, t); -} -function md5_hh(a, b, c, d, x, s, t) -{ - return md5_cmn(b ^ c ^ d, a, b, x, s, t); -} -function md5_ii(a, b, c, d, x, s, t) -{ - return md5_cmn(c ^ (b | (~d)), a, b, x, s, t); -} - -/* - * Add integers, wrapping at 2^32. This uses 16-bit operations internally - * to work around bugs in some JS interpreters. - */ -function safe_add(x, y) -{ - var lsw = (x & 0xFFFF) + (y & 0xFFFF); - var msw = (x >> 16) + (y >> 16) + (lsw >> 16); - return (msw << 16) | (lsw & 0xFFFF); -} - -/* - * Bitwise rotate a 32-bit number to the left. - */ -function bit_rol(num, cnt) -{ - return (num << cnt) | (num >>> (32 - cnt)); -} diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 1d004dc..3096448 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -22,7 +22,7 @@ THE SOFTWARE. */ -var md5 = require('./md5'); +var crypto = require("crypto"); var net = require("net"); var sys = require("sys"); var url = require('url'); @@ -644,12 +644,9 @@ function Prepared(sql, conn /*, use_named */) { // so we know when this gets tripped. callback(null, eP.results); } - else if (eP.noData === true) { - // Callback gets wrapped at the tx layer, - // so we know when this gets tripped. - callback(); - } else { + + console.dir(sys.inspect(callback)); callback(null, []); } }); @@ -830,7 +827,9 @@ function Connection(args) { // Set up callbacks to automatically do the login and other logic events.on('AuthenticationMD5Password', function (salt) { - var result = "md5" + md5(md5(args.password + args.username) + salt.toString("binary")); + var result = "md5" + crypto.createHash("md5").update( + crypto.createHash("md5").update(args.password + args.username).digest("hex") + salt.toString("binary") + ).digest("hex"); sendMessage('PasswordMessage', [result]); }); events.on('AuthenticationCleartextPassword', function () { diff --git a/package.json b/package.json index ec1a0d1..60c15c1 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ -{"name": "postgres", - "version": "0.2-dev", +{"name": "postgres-js", + "version": "0.1.0", "description": "Pure-JavaScript implementation of the Postgres Protocol", "homepage": "https://public.commandprompt.com/projects/postgres-js/", "author": { diff --git a/t/insert.js b/t/insert.js new file mode 100644 index 0000000..2f4864d --- /dev/null +++ b/t/insert.js @@ -0,0 +1,26 @@ +var sys = require("sys"); +var pg = require("../lib/postgres-pure"); +pg.DEBUG=3; + +var db = new pg.connect("pgsql://test:12345@localhost:5432/postgresjs"); +var tx = db.transaction(); +tx.begin(); +tx.query("CREATE TABLE testcase (id int)", function (err, rs) { + tx.query("INSERT INTO testcase VALUES (?)", 1, function(err, rs) { + console.log("Got to tx.") + }); + tx.query("DROP TABLE testcase;", function (err, rs) { + // nothing + }); + tx.rollback(); +}); +db.close(); + + + +function eq (l, r) { + if (l === r) { + return "ok" + } + return "not ok\n " + l + " != " + r; +} \ No newline at end of file From 44d655e040338954002a8e82e82672dff9f16f1f Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Sat, 12 Feb 2011 18:05:34 -0700 Subject: [PATCH 51/54] contact info changes in the readme. Removed the unused results.js. Bumped to 0.1.1, post changing to crypto. Works on node 0.4.0! --- README.md | 2 +- lib/results.js | 3 --- package.json | 6 +++--- 3 files changed, 4 insertions(+), 7 deletions(-) delete mode 100644 lib/results.js diff --git a/README.md b/README.md index ef8569b..684683d 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This library allows for the correct handling of server-side prepared queries. Nested DB calls will be executed in the order of definition. -All code is available under the terms of the MIT license, unless otherwise noted (md5.js) +All code is available under the terms of the MIT license (c) 2010, Tim Caswell, Aurynn Shaw. diff --git a/lib/results.js b/lib/results.js deleted file mode 100644 index 97e28de..0000000 --- a/lib/results.js +++ /dev/null @@ -1,3 +0,0 @@ -function result (desc, row) { - this.description = desc; -} \ No newline at end of file diff --git a/package.json b/package.json index 60c15c1..91fa753 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,10 @@ {"name": "postgres-js", - "version": "0.1.0", - "description": "Pure-JavaScript implementation of the Postgres Protocol", + "version": "0.1.1", + "description": "Pure-JavaScript implementation of the Postgres protocol", "homepage": "https://public.commandprompt.com/projects/postgres-js/", "author": { "name": "Aurynn Shaw", - "email": "ashaw@commandprompt.com", + "email": "aurynn@gmail.com", }, "main": "lib/postgres-pure", "dependencies" : From 000ef214707838258d38f2a3a6d9fe5d28e76b12 Mon Sep 17 00:00:00 2001 From: Augusto Pascutti Date: Thu, 3 Mar 2011 17:44:21 -0300 Subject: [PATCH 52/54] Fixed date/timestamp column type being interpreted wrongly. --- lib/postgres-pure.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 3096448..87a5a61 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -1030,8 +1030,10 @@ function Connection(args) { break; case 1082: // Date value = new Date(value); + break; case 1114: // Timestamp, no timezone value = new Date(value); + break; case 1184: // Timestamp, with timezone // Initial value: // "2011-02-01 21:00:52.353444-07" @@ -1043,7 +1045,6 @@ function Connection(args) { var tz = value.slice(0, value.length-1); tz = tz.slice(0, tz.length-4); value = new Date(tz + " GMT+0000" ); - console.log(value); } else { var tz = value.slice(value.length-3, value.length); // last three. From eb8e31a6801fb7be37dd271b00d14ed52f52a4fe Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Thu, 3 Mar 2011 17:20:44 -0700 Subject: [PATCH 53/54] Adding in rvanvelzen's isNaN() patch, line 750. Adding in augustohp's break fixes for timestamp checking. --- lib/postgres-pure.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/postgres-pure.js b/lib/postgres-pure.js index 87a5a61..5edba31 100644 --- a/lib/postgres-pure.js +++ b/lib/postgres-pure.js @@ -747,7 +747,7 @@ function Connection(args) { var length = first.int32Read(1) - 4; // Make sure we have a whole message, TCP comes in chunks - if (first.length < length + 5) { + if (first.length < length + 5 || isNaN(length)) { if (queue.length > 1) { // Merge the first two buffers queue.shift(); From d13ee3de099a19c4c2f4c32bd44ae8e847765d28 Mon Sep 17 00:00:00 2001 From: Anthony Sekatski Date: Sat, 2 Apr 2011 07:17:05 -0700 Subject: [PATCH 54/54] Fix package.json for NPM version >= 0.3 (author - string, contributors - array) --- package.json | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 91fa753..ee1f816 100644 --- a/package.json +++ b/package.json @@ -2,10 +2,11 @@ "version": "0.1.1", "description": "Pure-JavaScript implementation of the Postgres protocol", "homepage": "https://public.commandprompt.com/projects/postgres-js/", - "author": { - "name": "Aurynn Shaw", - "email": "aurynn@gmail.com", - }, + "author": "Tim Caswell ", + "contributors": [ + { "name": "Aurynn Shaw","email": "aurynn@gmail.com" }, + { "name": "Anthony Sekatski","email": "anthony@8protons.com" }, + ], "main": "lib/postgres-pure", "dependencies" : { "strtok" : ">=0.1.1"