Skip to content

Commit

Permalink
Refactor js parser
Browse files Browse the repository at this point in the history 
Fix tests to work with Node.js 0.10
Improve average use case speed by up to 20%
Fix some small js parser issues
  • Loading branch information
Ruben Bridgewater committed Nov 23, 2015
1 parent b6a81a4 commit ac9ff9a
Show file tree
Hide file tree
Showing 4 changed files with 172 additions and 130 deletions.
70 changes: 35 additions & 35 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ redis - a node.js redis client
[![Windows Tests](https://img.shields.io/appveyor/ci/BridgeAR/node-redis/master.svg?label=Windows%20Tests)](https://ci.appveyor.com/project/BridgeAR/node-redis/branch/master)
[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/NodeRedis/node_redis?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)

This is a complete and feature rich Redis client for node.js. It supports all Redis commands and focuses on performance.
This is a complete and feature rich Redis client for node.js. It supports all Redis commands and focuses on high performance.

Install with:

Expand Down Expand Up @@ -646,40 +646,40 @@ hiredis parser (Lenovo T450s i7-5600U):

```
Client count: 1, node version: 4.2.2, server version: 3.0.3, parser: hiredis
PING, 1/1 min/max/avg/p95: 0/ 3/ 0.02/ 0.00 2501ms total, 39862.85 ops/sec
PING, batch 50/1 min/max/avg/p95: 0/ 2/ 0.10/ 1.00 2501ms total, 491223.51 ops/sec
SET 4B str, 1/1 min/max/avg/p95: 0/ 3/ 0.03/ 0.00 2501ms total, 36387.45 ops/sec
SET 4B str, batch 50/1 min/max/avg/p95: 0/ 3/ 0.14/ 1.00 2501ms total, 346381.45 ops/sec
SET 4B buf, 1/1 min/max/avg/p95: 0/ 2/ 0.04/ 0.00 2501ms total, 24395.84 ops/sec
SET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 156457.42 ops/sec
GET 4B str, 1/1 min/max/avg/p95: 0/ 3/ 0.03/ 0.00 2501ms total, 36906.44 ops/sec
GET 4B str, batch 50/1 min/max/avg/p95: 0/ 3/ 0.12/ 1.00 2501ms total, 425729.71 ops/sec
GET 4B buf, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 36221.91 ops/sec
GET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.11/ 1.00 2501ms total, 430407.84 ops/sec
SET 4KiB str, 1/1 min/max/avg/p95: 0/ 3/ 0.03/ 0.00 2501ms total, 30951.22 ops/sec
SET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.33/ 1.00 2501ms total, 150299.88 ops/sec
SET 4KiB buf, 1/1 min/max/avg/p95: 0/ 2/ 0.04/ 1.00 2501ms total, 23919.63 ops/sec
SET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.36/ 1.00 2501ms total, 139204.32 ops/sec
GET 4KiB str, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 32739.30 ops/sec
GET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 154158.34 ops/sec
GET 4KiB buf, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 34654.94 ops/sec
GET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 153758.50 ops/sec
INCR, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 37530.19 ops/sec
INCR, batch 50/1 min/max/avg/p95: 0/ 3/ 0.12/ 1.00 2501ms total, 415993.60 ops/sec
LPUSH, 1/1 min/max/avg/p95: 0/ 1/ 0.03/ 0.00 2501ms total, 37409.04 ops/sec
LPUSH, batch 50/1 min/max/avg/p95: 0/ 2/ 0.14/ 1.00 2501ms total, 354778.09 ops/sec
LRANGE 10, 1/1 min/max/avg/p95: 0/ 3/ 0.03/ 0.00 2501ms total, 31768.49 ops/sec
LRANGE 10, batch 50/1 min/max/avg/p95: 0/ 3/ 0.33/ 1.00 2501ms total, 151379.45 ops/sec
LRANGE 100, 1/1 min/max/avg/p95: 0/ 2/ 0.06/ 1.00 2501ms total, 16801.68 ops/sec
LRANGE 100, batch 50/1 min/max/avg/p95: 2/ 4/ 2.07/ 3.00 2501ms total, 24150.34 ops/sec
SET 4MiB str, 1/1 min/max/avg/p95: 1/ 5/ 1.96/ 2.00 2501ms total, 510.20 ops/sec
SET 4MiB str, batch 20/1 min/max/avg/p95: 83/ 108/ 94.44/ 106.40 2550ms total, 211.76 ops/sec
SET 4MiB buf, 1/1 min/max/avg/p95: 1/ 7/ 2.06/ 3.00 2501ms total, 484.21 ops/sec
SET 4MiB buf, batch 20/1 min/max/avg/p95: 38/ 48/ 40.90/ 46.00 2536ms total, 488.96 ops/sec
GET 4MiB str, 1/1 min/max/avg/p95: 3/ 13/ 5.20/ 9.00 2503ms total, 192.17 ops/sec
GET 4MiB str, batch 20/1 min/max/avg/p95: 74/ 105/ 87.24/ 104.00 2530ms total, 229.25 ops/sec
GET 4MiB buf, 1/1 min/max/avg/p95: 3/ 11/ 5.01/ 9.00 2501ms total, 199.12 ops/sec
GET 4MiB buf, batch 20/1 min/max/avg/p95: 78/ 93/ 84.23/ 91.90 2528ms total, 237.34 ops/sec
PING, 1/1 min/max/avg/p95: 0/ 2/ 0.02/ 0.00 2501ms total, 47503.80 ops/sec
PING, batch 50/1 min/max/avg/p95: 0/ 2/ 0.09/ 1.00 2501ms total, 529668.13 ops/sec
SET 4B str, 1/1 min/max/avg/p95: 0/ 2/ 0.02/ 0.00 2501ms total, 41900.04 ops/sec
SET 4B str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.14/ 1.00 2501ms total, 354658.14 ops/sec
SET 4B buf, 1/1 min/max/avg/p95: 0/ 4/ 0.04/ 0.00 2501ms total, 23499.00 ops/sec
SET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.31/ 1.00 2501ms total, 159836.07 ops/sec
GET 4B str, 1/1 min/max/avg/p95: 0/ 4/ 0.02/ 0.00 2501ms total, 43489.80 ops/sec
GET 4B str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.11/ 1.00 2501ms total, 444202.32 ops/sec
GET 4B buf, 1/1 min/max/avg/p95: 0/ 3/ 0.02/ 0.00 2501ms total, 38561.38 ops/sec
GET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.11/ 1.00 2501ms total, 452139.14 ops/sec
SET 4KiB str, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 32990.80 ops/sec
SET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.34/ 1.00 2501ms total, 146161.54 ops/sec
SET 4KiB buf, 1/1 min/max/avg/p95: 0/ 1/ 0.04/ 0.00 2501ms total, 23294.28 ops/sec
SET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.36/ 1.00 2501ms total, 137584.97 ops/sec
GET 4KiB str, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 36350.66 ops/sec
GET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 155157.94 ops/sec
GET 4KiB buf, 1/1 min/max/avg/p95: 0/ 4/ 0.02/ 0.00 2501ms total, 39776.49 ops/sec
GET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 155457.82 ops/sec
INCR, 1/1 min/max/avg/p95: 0/ 3/ 0.02/ 0.00 2501ms total, 43972.41 ops/sec
INCR, batch 50/1 min/max/avg/p95: 0/ 1/ 0.12/ 1.00 2501ms total, 425809.68 ops/sec
LPUSH, 1/1 min/max/avg/p95: 0/ 2/ 0.02/ 0.00 2501ms total, 38998.40 ops/sec
LPUSH, batch 50/1 min/max/avg/p95: 0/ 4/ 0.14/ 1.00 2501ms total, 365013.99 ops/sec
LRANGE 10, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 31879.25 ops/sec
LRANGE 10, batch 50/1 min/max/avg/p95: 0/ 1/ 0.32/ 1.00 2501ms total, 153698.52 ops/sec
LRANGE 100, 1/1 min/max/avg/p95: 0/ 4/ 0.06/ 0.00 2501ms total, 16676.13 ops/sec
LRANGE 100, batch 50/1 min/max/avg/p95: 1/ 6/ 2.03/ 2.00 2502ms total, 24520.38 ops/sec
SET 4MiB str, 1/1 min/max/avg/p95: 1/ 6/ 2.11/ 3.00 2502ms total, 472.82 ops/sec
SET 4MiB str, batch 20/1 min/max/avg/p95: 85/ 112/ 94.93/ 109.60 2563ms total, 210.69 ops/sec
SET 4MiB buf, 1/1 min/max/avg/p95: 1/ 8/ 2.02/ 3.00 2502ms total, 490.01 ops/sec
SET 4MiB buf, batch 20/1 min/max/avg/p95: 37/ 52/ 39.48/ 46.75 2528ms total, 506.33 ops/sec
GET 4MiB str, 1/1 min/max/avg/p95: 3/ 13/ 5.26/ 9.00 2504ms total, 190.10 ops/sec
GET 4MiB str, batch 20/1 min/max/avg/p95: 70/ 106/ 89.36/ 103.75 2503ms total, 223.73 ops/sec
GET 4MiB buf, 1/1 min/max/avg/p95: 3/ 11/ 5.04/ 8.15 2502ms total, 198.24 ops/sec
GET 4MiB buf, batch 20/1 min/max/avg/p95: 70/ 105/ 88.07/ 103.00 2554ms total, 227.09 ops/sec
```

The hiredis and js parser should most of the time be on the same level. But if you use Redis for big SUNION/SINTER/LRANGE/ZRANGE hiredis is significantly faster.
Expand Down
8 changes: 2 additions & 6 deletions index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,14 +294,10 @@ RedisClient.prototype.init_parser = function () {
// Important: Only send results / errors async.
// That way the result / error won't stay in a try catch block and catch user things
this.reply_parser.send_error = function (data) {
process.nextTick(function() {
self.return_error(data);
});
self.return_error(data);
};
this.reply_parser.send_reply = function (data) {
process.nextTick(function() {
self.return_reply(data);
});
self.return_reply(data);
};
};

Expand Down
116 changes: 50 additions & 66 deletions lib/parsers/javascript.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ function JavascriptReplyParser() {
this._big_offset = 0;
this._chunks_size = 0;
this._buffers = [];
this._type = 0;
this._protocol_error = false;
}

function IncompleteReadBuffer(message) {
Expand All @@ -21,87 +23,73 @@ JavascriptReplyParser.prototype._parseResult = function (type) {
var start = 0,
end = 0,
offset = 0,
packetHeader = 0;
packetHeader = 0,
res,
reply;

if (type === 43 || type === 58 || type === 45) { // + or : or -
// up to the delimiter
// Up to the delimiter
end = this._packetEndOffset();
start = this._offset;

// include the delimiter
// Include the delimiter
this._offset = end + 2;

if (type === 43) {
return this._buffer.slice(start, end);
} else if (type === 58) {
// return the coerced numeric value
// Return the coerced numeric value
return +this._buffer.toString('ascii', start, end);
}
return new Error(this._buffer.toString('utf-8', start, end));
} else if (type === 36) { // $
packetHeader = this.parseHeader();

// packets with a size of -1 are considered null
// Packets with a size of -1 are considered null
if (packetHeader === -1) {
return null;
}

end = this._offset + packetHeader;
start = this._offset;

if (end > this._buffer.length) {
if (end + 2 > this._buffer.length) {
this._chunks_size = this._buffer.length - this._offset - 2;
this._big_offset = packetHeader;
throw new IncompleteReadBuffer('Wait for more data.');
}

// set the offset to after the delimiter
// Set the offset to after the delimiter
this._offset = end + 2;

return this._buffer.slice(start, end);
} else if (type === 42) { // *
// set a rewind point, as the packet is larger than the buffer in memory
// Set a rewind point, as the packet is larger than the buffer in memory
offset = this._offset;
packetHeader = this.parseHeader();

if (packetHeader === -1) {
return null;
}

if (packetHeader > this._buffer.length - this._offset) {
this._offset = offset - 1;
throw new IncompleteReadBuffer('Wait for more data.');
}

var reply = [];
var ntype, i, res;

reply = [];
offset = this._offset - 1;

for (i = 0; i < packetHeader; i++) {
ntype = this._buffer[this._offset++];

if (this._offset > this._buffer.length) {
for (var i = 0; i < packetHeader; i++) {
if (this._offset >= this._buffer.length) {
throw new IncompleteReadBuffer('Wait for more data.');
}
res = this._parseResult(ntype);
res = this._parseResult(this._buffer[this._offset++]);
reply.push(res);
}

return reply;
} else {
return null;
return void 0;
}
};

JavascriptReplyParser.prototype.execute = function (buffer) {

if (this._chunks_size !== 0 && this._big_offset > this._chunks_size + buffer.length) {
this._buffers.push(buffer);
this._chunks_size += buffer.length;
return;
}

if (this._buffers.length !== 0) {
this._buffers.unshift(this._offset === 0 ? this._buffer : this._buffer.slice(this._offset));
this._buffers.push(buffer);
Expand All @@ -115,44 +103,41 @@ JavascriptReplyParser.prototype.execute = function (buffer) {
this._buffer = Buffer.concat([this._buffer.slice(this._offset), buffer]);
}
this._offset = 0;
this._protocol_error = true;
this.run();
};

JavascriptReplyParser.prototype.run = function (buffer) {
var type, offset = this._offset;

while (true) {
offset = this._offset;
// at least 4 bytes: :1\r\n
if (this._buffer.length - this._offset < 4) {
break;
}
JavascriptReplyParser.prototype.try_parsing = function () {
// Set a rewind point. If a failure occurs, wait for the next execute()/append() and try again
var offset = this._offset - 1;
try {
return this._parseResult(this._type);
} catch (err) {
// Catch the error (not enough data), rewind if it's an array,
// and wait for the next packet to appear
this._offset = offset;
this._protocol_error = false;
return void 0;
}
};

try {
type = this._buffer[this._offset++];

if (type === 43 || type === 58 || type === 36) { // Strings + // Integers : // Bulk strings $
this.send_reply(this._parseResult(type));
} else if (type === 45) { // Errors -
this.send_error(this._parseResult(type));
} else if (type === 42) { // Arrays *
// set a rewind point. if a failure occurs,
// wait for the next execute()/append() and try again
offset = this._offset - 1;

this.send_reply(this._parseResult(type));
} else if (type !== 10 && type !== 13) {
// Reset the buffer so the parser can handle following commands properly
this._buffer = new Buffer(0);
var err = new Error('Protocol error, got "' + String.fromCharCode(type) + '" as reply type byte');
this.send_error(err);
}
} catch (err) {
// catch the error (not enough data), rewind, and wait
// for the next packet to appear
this._offset = offset;
break;
JavascriptReplyParser.prototype.run = function (buffer) {
this._type = this._buffer[this._offset++];
var reply = this.try_parsing();

while (reply !== undefined) {
if (this._type === 45) { // Errors -
this.send_error(reply);
} else {
this.send_reply(reply); // Strings + // Integers : // Bulk strings $ // Arrays *
}
this._type = this._buffer[this._offset++];
reply = this.try_parsing();
}
if (this._type !== undefined && this._protocol_error === true) {
// Reset the buffer so the parser can handle following commands properly
this._buffer = new Buffer(0);
this.send_error(new Error('Protocol error, got "' + String.fromCharCode(this._type) + '" as reply type byte'));
}
};

Expand All @@ -161,21 +146,20 @@ JavascriptReplyParser.prototype.parseHeader = function () {
value = this._buffer.toString('ascii', this._offset, end) | 0;

this._offset = end + 2;

return value;
};

JavascriptReplyParser.prototype._packetEndOffset = function () {
var offset = this._offset;
var offset = this._offset,
len = this._buffer.length - 1;

while (this._buffer[offset] !== 0x0d && this._buffer[offset + 1] !== 0x0a) {
offset++;

if (offset >= this._buffer.length) {
if (offset >= len) {
throw new IncompleteReadBuffer('Did not see LF after NL reading multi bulk count (' + offset + ' => ' + this._buffer.length + ', ' + this._offset + ')');
}
}

return offset;
};

Expand Down
Loading

0 comments on commit ac9ff9a

Please sign in to comment.