Skip to content

Commit

Permalink
Use a more robust persistence scheme, thanks spolu and szawcz
Browse files Browse the repository at this point in the history
  • Loading branch information
@louischatriot
louischatriot committed Nov 28, 2013
1 parent 35b46c2 commit 4776c98
Show file tree
Hide file tree
Showing 4 changed files with 137 additions and 98 deletions.
17 changes: 16 additions & 1 deletion lib/customUtils.js
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
var crypto = require('crypto');
var crypto = require('crypto')
, fs = require('fs')
;

/**
* Return a random alphanumerical string of length len
Expand All @@ -16,4 +18,17 @@ function uid (len) {
}


/**
* Callback signature: err
*/
function ensureFileDoesntExist (file, callback) {
fs.exists(file, function (exists) {
if (!exists) { return callback(null); }

fs.unlink(file, function (err) { return callback(err); });
});
}


module.exports.uid = uid;
module.exports.ensureFileDoesntExist = ensureFileDoesntExist;
68 changes: 18 additions & 50 deletions lib/persistence.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ var fs = require('fs')
, model = require('./model')
, async = require('async')
, mkdirp = require('mkdirp')
, customUtils = require('./customUtils')
;


Expand All @@ -29,6 +30,7 @@ function Persistence (options) {
throw "The datafile name can't end with a ~, which is reserved for automatic backup files";
} else {
this.tempFilename = this.filename + '~';
this.oldFilename = this.filename + '~~';
}
}

Expand Down Expand Up @@ -103,36 +105,24 @@ Persistence.prototype.persistCachedDatabase = function (cb) {
});

async.waterfall([
function (cb) {
fs.exists(self.tempFilename, function (exists) {
if (exists) { // Shouldn't happen since ensureDatafileIntegrity removes the temp datafile
fs.unlink(self.tempFilename, function (err) { return cb(err); });
} else {
return cb();
}
});
}
async.apply(customUtils.ensureFileDoesntExist, self.tempFilename)
, async.apply(customUtils.ensureFileDoesntExist, self.oldFilename)
, function (cb) {
fs.exists(self.filename, function (exists) {
if (exists) {
fs.rename(self.filename, self.tempFilename, function (err) { return cb(err); });
fs.rename(self.filename, self.oldFilename, function (err) { return cb(err); });
} else {
return cb();
}
});
}
, function (cb) {
fs.writeFile(self.filename, toPersist, function (err) { return cb(err); });
fs.writeFile(self.tempFilename, toPersist, function (err) { return cb(err); });
}
, function (cb) {
fs.exists(self.tempFilename, function (exists) {
if (exists) {
fs.unlink(self.tempFilename, function (err) { return cb(err); });
} else {
return cb();
}
});
fs.rename(self.tempFilename, self.filename, function (err) { return cb(err); });
}
, async.apply(customUtils.ensureFileDoesntExist, self.oldFilename)
], function (err) { if (err) { return callback(err); } else { return callback(null); } })
};

Expand Down Expand Up @@ -236,41 +226,19 @@ Persistence.treatRawData = function (rawData) {
*/
Persistence.prototype.ensureDatafileIntegrity = function (callback) {
var self = this ;

fs.exists(self.filename, function (filenameExists) {
fs.exists(self.tempFilename, function (tempFilenameExists) {
// Normal case
if (filenameExists && !tempFilenameExists) {
return callback(null);
// Write was successful
if (filenameExists) { return callback(null); }

fs.exists(self.oldFilename, function (oldFilenameExists) {
// New database
if (!oldFilenameExists) {
return fs.writeFile(self.filename, '', 'utf8', function (err) { callback(err); });
}

// Process crashed right after renaming filename
if (!filenameExists && tempFilenameExists) {
return fs.rename(self.tempFilename, self.filename, function (err) { return callback(err); });
}

// No file exists, create empty datafile
if (!filenameExists && !tempFilenameExists) {
return fs.writeFile(self.filename, '', 'utf8', function (err) { callback(err); });
}

// Process crashed after or during write to datafile
// If datafile is not empty, it means process crashed after the write so we use it
// If it is empty, we don't know whether the database was emptied or we had a crash during write. The safest option is to use the temp datafile
if (filenameExists && tempFilenameExists) {
return fs.stat(self.filename, function (err, stats) {
if (err) { return callback(err); }

if (stats.size > 0) {
fs.unlink(self.tempFilename, function (err) { return callback(err); });
} else {
fs.unlink(self.filename, function (err) {
if (err) { return callback(err); }
fs.rename(self.tempFilename, self.filename, function (err) { return callback(err); });
});
}
});
}
// Write failed, use old version
fs.rename(self.oldFilename, self.filename, function (err) { return callback(err); });
});
});
};
Expand Down
35 changes: 35 additions & 0 deletions test/customUtil.test.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
var should = require('chai').should()
, assert = require('chai').assert
, customUtils = require('../lib/customUtils')
, fs = require('fs')
;


describe('customUtils', function () {

describe('ensureFileDoesntExist', function () {

it('Doesnt do anything if file already doesnt exist', function (done) {
customUtils.ensureFileDoesntExist('workspace/nonexisting', function (err) {
assert.isNull(err);
fs.existsSync('workspace/nonexisting').should.equal(false);
done();
});
});

it('Deletes file if it exists', function (done) {
fs.writeFileSync('workspace/existing', 'hello world', 'utf8');
fs.existsSync('workspace/existing').should.equal(true);

customUtils.ensureFileDoesntExist('workspace/existing', function (err) {
assert.isNull(err);
fs.existsSync('workspace/existing').should.equal(false);
done();
});
});

});



});
115 changes: 68 additions & 47 deletions test/persistence.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,15 +265,16 @@ describe('Persistence', function () {
var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });

if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~')) { fs.unlinkSync('workspace/it.db~'); }
if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }

fs.existsSync('workspace/it.db').should.equal(false);
fs.existsSync('workspace/it.db~').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(false);

p.ensureDatafileIntegrity(function (err) {
assert.isNull(err);

fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(false);

fs.readFileSync('workspace/it.db', 'utf8').should.equal('');

Expand All @@ -285,95 +286,79 @@ describe('Persistence', function () {
var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });

if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~')) { fs.unlinkSync('workspace/it.db~'); }
if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }

fs.writeFileSync('workspace/it.db', 'something', 'utf8');

fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(false);

p.ensureDatafileIntegrity(function (err) {
assert.isNull(err);

fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(false);

fs.readFileSync('workspace/it.db', 'utf8').should.equal('something');

done();
});
});

it('If only temp datafile exists, ensureDatafileIntegrity will use it', function (done) {
it('If old datafile exists and datafile doesnt, ensureDatafileIntegrity will use it', function (done) {
var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });

if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~')) { fs.unlinkSync('workspace/it.db~'); }
if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }

fs.writeFileSync('workspace/it.db~', 'something', 'utf8');
fs.writeFileSync('workspace/it.db~~', 'something', 'utf8');

fs.existsSync('workspace/it.db').should.equal(false);
fs.existsSync('workspace/it.db~').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(true);

p.ensureDatafileIntegrity(function (err) {
assert.isNull(err);

fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(false);

fs.readFileSync('workspace/it.db', 'utf8').should.equal('something');

done();
});
});

it('If both files exist and datafile is not empty, ensureDatafileIntegrity will use the datafile', function (done) {
var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });

if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~')) { fs.unlinkSync('workspace/it.db~'); }

fs.writeFileSync('workspace/it.db', 'something', 'utf8');
fs.writeFileSync('workspace/it.db~', 'other', 'utf8');

fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(true);

p.ensureDatafileIntegrity(function (err) {
assert.isNull(err);

fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(false);

fs.readFileSync('workspace/it.db', 'utf8').should.equal('something');

done();
});
});

it('If both files exist and datafile is empty, ensureDatafileIntegrity will use the temp datafile', function (done) {
var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
it('If both old and current datafiles exist, ensureDatafileIntegrity will use the datafile, it means step 1 of persistence failed', function (done) {
var theDb = new Datastore({ filename: 'workspace/it.db' });

if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~')) { fs.unlinkSync('workspace/it.db~'); }
if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }

fs.writeFileSync('workspace/it.db', '', 'utf8');
fs.writeFileSync('workspace/it.db~', 'other', 'utf8');
fs.writeFileSync('workspace/it.db', '{"_id":"0","hello":"world"}', 'utf8');
fs.writeFileSync('workspace/it.db~~', '{"_id":"0","hello":"other"}', 'utf8');

fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(true);

p.ensureDatafileIntegrity(function (err) {
theDb.persistence.ensureDatafileIntegrity(function (err) {
assert.isNull(err);

fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(true);

fs.readFileSync('workspace/it.db', 'utf8').should.equal('other');
fs.readFileSync('workspace/it.db', 'utf8').should.equal('{"_id":"0","hello":"world"}');

done();
theDb.loadDatabase(function (err) {
assert.isNull(err);
theDb.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(1);
docs[0].hello.should.equal("world");
done();
});
});
});
});
});

it('persistCachedDatabase should update the contents of the datafile and leave a clean state', function (done) {
d.insert({ hello: 'world' }, function () {
Expand All @@ -382,14 +367,17 @@ describe('Persistence', function () {

if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
if (fs.existsSync(testDb + '~')) { fs.unlinkSync(testDb + '~'); }
if (fs.existsSync(testDb + '~~')) { fs.unlinkSync(testDb + '~~'); }
fs.existsSync(testDb).should.equal(false);
fs.existsSync(testDb + '~').should.equal(false);
fs.existsSync(testDb + '~~').should.equal(false);

d.persistence.persistCachedDatabase(function (err) {
var contents = fs.readFileSync(testDb, 'utf8');
assert.isNull(err);
fs.existsSync(testDb).should.equal(true);
fs.existsSync(testDb + '~').should.equal(false);
fs.existsSync(testDb + '~~').should.equal(false);
if (!contents.match(/^{"hello":"world","_id":"[0-9a-zA-Z]{16}"}\n$/)) {
throw "Datafile contents not as expected";
}
Expand All @@ -399,21 +387,54 @@ describe('Persistence', function () {
});
});

it('persistCachedDatabase should update the contents of the datafile and leave a clean state even if there is a temp datafile', function (done) {
it('After a persistCachedDatabase, there should be no temp or old filename', function (done) {
d.insert({ hello: 'world' }, function () {
d.find({}, function (err, docs) {
docs.length.should.equal(1);

if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
if (fs.existsSync(testDb + '~')) { fs.unlinkSync(testDb + '~'); }
if (fs.existsSync(testDb + '~~')) { fs.unlinkSync(testDb + '~~'); }
fs.existsSync(testDb).should.equal(false);

fs.writeFileSync(testDb + '~', 'bloup', 'utf8');
fs.writeFileSync(testDb + '~~', 'blap', 'utf8');
fs.existsSync(testDb + '~').should.equal(true);
fs.existsSync(testDb + '~~').should.equal(true);

d.persistence.persistCachedDatabase(function (err) {
var contents = fs.readFileSync(testDb, 'utf8');
assert.isNull(err);
fs.existsSync(testDb).should.equal(true);
fs.existsSync(testDb + '~').should.equal(false);
fs.existsSync(testDb + '~~').should.equal(false);
if (!contents.match(/^{"hello":"world","_id":"[0-9a-zA-Z]{16}"}\n$/)) {
throw "Datafile contents not as expected";
}
done();
});
});
});
});

it('persistCachedDatabase should update the contents of the datafile and leave a clean state even if there is a temp or old datafile', function (done) {
d.insert({ hello: 'world' }, function () {
d.find({}, function (err, docs) {
docs.length.should.equal(1);

if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
fs.writeFileSync(testDb + '~', 'blabla', 'utf8');
fs.writeFileSync(testDb + '~~', 'bloblo', 'utf8');
fs.existsSync(testDb).should.equal(false);
fs.existsSync(testDb + '~').should.equal(true);
fs.existsSync(testDb + '~~').should.equal(true);

d.persistence.persistCachedDatabase(function (err) {
var contents = fs.readFileSync(testDb, 'utf8');
assert.isNull(err);
fs.existsSync(testDb).should.equal(true);
fs.existsSync(testDb + '~').should.equal(false);
fs.existsSync(testDb + '~~').should.equal(false);
if (!contents.match(/^{"hello":"world","_id":"[0-9a-zA-Z]{16}"}\n$/)) {
throw "Datafile contents not as expected";
}
Expand Down

0 comments on commit 4776c98

Please sign in to comment.