From 8d1bc716492687a001acdded8f11002b2dbddf27 Mon Sep 17 00:00:00 2001 From: Mark Moffat Date: Thu, 25 May 2017 13:19:15 +0930 Subject: [PATCH] Fix #171 - Moving to a stored Lunr index and upgrading to latest version (2.0.4). --- .travis.yml | 6 +- README.md | 2 + app.js | 84 +-------------- package.json | 4 +- public/javascripts/openKB.js | 28 ++--- routes/api.js | 28 +---- routes/common.js | 79 +++++++++++++- routes/index.js | 197 +++++++++++++---------------------- views/layouts/layout.hbs | 2 +- 9 files changed, 180 insertions(+), 250 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5ab1ddc5..19ff8f73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,6 @@ language: node_js node_js: - "node" - "iojs" - - "0.12" - - "0.11" - - "0.10" \ No newline at end of file + - "6" + - "4" + - "0.12" \ No newline at end of file diff --git a/README.md b/README.md index aac419b0..84d0fe28 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ Demo: [http://openkb.markmoffat.com](http://openkb.markmoffat.com) 3. Start application: `npm start` 4. Go to [http://127.0.0.1:4444](http://127.0.0.1:4444) in your browser +> Note: `openKB` supports Nodejs version 0.12 and above. + ### Deploy on Heroku [![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy?template=https://github.com/mrvautin/openKB) diff --git a/app.js b/app.js index ae0010c0..f134130b 100644 --- a/app.js +++ b/app.js @@ -7,7 +7,6 @@ var bodyParser = require('body-parser'); var Nedb = require('nedb'); var session = require('express-session'); var bcrypt = require('bcrypt-nodejs'); -var lunr = require('lunr'); var markdownit = require('markdown-it')({html: true, linkify: true, typographer: true}); var moment = require('moment'); var fs = require('fs'); @@ -18,7 +17,6 @@ var config = common.read_config(); var MongoClient = require('mongodb').MongoClient; var expstate = require('express-state'); var compression = require('compression'); -var lunr_store = {}; // require the routes var index = require('./routes/index'); @@ -224,8 +222,6 @@ app.use(function (req, res, next){ req.handlebars = handlebars.helpers; req.bcrypt = bcrypt; req.i18n = i18n; - req.lunr_index = lunr_index; - req.lunr_store = lunr_store; req.app_context = app_context; req.i18n.setLocaleFromCookie(); next(); @@ -290,7 +286,7 @@ if(config.settings.database.type === 'embedded'){ app.db = db; // add articles to index - indexArticles(db, function(err){ + common.buildIndex(db, function(err){ // lift the app app.listen(app.get('port'), app.get('bind'), function (){ console.log('openKB running on host: http://' + app.get('bind') + ':' + app.get('port')); @@ -314,7 +310,7 @@ if(config.settings.database.type === 'embedded'){ app.db = db; // add articles to index - indexArticles(db, function(err){ + common.buildIndex(db, function(err){ // lift the app app.listen(app.get('port'), app.get('bind'), function (){ console.log('openKB running on host: http://' + app.get('bind') + ':' + app.get('port')); @@ -324,80 +320,4 @@ if(config.settings.database.type === 'embedded'){ }); } -// setup lunr indexing -var lunr_index = lunr(function (){ - this.field('kb_title', {boost: 10}); - this.field('kb_keywords'); -}); - -// if index body is switched on -if(config.settings.index_article_body === true){ - lunr_index.field('kb_body'); -} - -function indexArticles(db, callback){ - // get all articles on startup - if(config.settings.database.type === 'embedded'){ - db.kb.find({kb_published: 'true'}, function (err, kb_list){ - // add to lunr index - kb_list.forEach(function(kb){ - // only if defined - var keywords = ''; - if(kb.kb_keywords !== undefined){ - keywords = kb.kb_keywords.toString().replace(/,/g, ' '); - } - - var doc = { - 'kb_title': kb.kb_title, - 'kb_keywords': keywords, - 'id': kb._id - }; - - // if index body is switched on - if(config.settings.index_article_body === true){ - doc['kb_body'] = kb.kb_body; - } - - // add to store - var href = kb.kb_permalink !== '' ? kb.kb_permalink : kb._id; - lunr_store[kb._id] = {t: kb.kb_title, p: href}; - - lunr_index.add(doc); - }); - - callback(null); - }); - }else{ - db.kb.find({kb_published: 'true'}).toArray(function (err, kb_list){ - // add to lunr index - kb_list.forEach(function(kb){ - // only if defined - var keywords = ''; - if(typeof kb.kb_keywords !== 'undefined' && kb.kb_keywords !== null){ - keywords = kb.kb_keywords.toString().replace(/,/g, ' '); - } - - var doc = { - 'kb_title': kb.kb_title, - 'kb_keywords': keywords, - 'id': kb._id - }; - - // if index body is switched on - if(config.settings.index_article_body === true){ - doc['kb_body'] = kb.kb_body; - } - - // add to store - var href = kb.kb_permalink !== '' ? kb.kb_permalink : kb._id; - lunr_store[kb._id] = {t: kb.kb_title, p: href}; - - lunr_index.add(doc); - }); - - callback(null); - }); - } -} - module.exports = app; diff --git a/package.json b/package.json index d075e73a..e6cd7bf4 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "jszip": "^2.6.1", "junk": "^2.0.0", "lodash": "^4.17.2", - "lunr": "^0.7.1", + "lunr": "^2.0.4", "markdown-it": "^7.0.1", "mime-types": "^2.1.13", "mkdirp": "^0.5.1", @@ -72,5 +72,5 @@ "bugs": { "url": "https://github.com/mrvautin/openKB/issues" }, - "homepage": "http://openkb.mrvautin.com" + "homepage": "https://openkb.markmoffat.com" } diff --git a/public/javascripts/openKB.js b/public/javascripts/openKB.js index 00ee7eb4..1e9d94f8 100644 --- a/public/javascripts/openKB.js +++ b/public/javascripts/openKB.js @@ -48,23 +48,25 @@ $(document).ready(function(){ data: {id: this.id, state: this.checked} }) .done(function(response){ - var index = lunr.Index.load(response.index); - var store = response.store; + var index = lunr.Index.load(JSON.parse(response.index)); + var store = JSON.parse(response.store); $('#frm_search').on('keyup', function(){ var query = $(this).val(); - var results = index.search(query); - if(results.length === 0){ - $('#searchResult').addClass('hidden'); - }else{ - $('.searchResultList').empty(); - $('.searchResultList').append('
  • Search results
    • '); - for(var result in results){ - var ref = results[result].ref; - var searchitem = '
  • ' + store[ref].t + '
    • '; - $('.searchResultList').append(searchitem); + if(query.length > 2){ + var results = index.search(query); + if(results.length === 0){ + $('#searchResult').addClass('hidden'); + }else{ + $('.searchResultList').empty(); + $('.searchResultList').append('
  • Search results
    • '); + for(var result in results){ + var ref = results[result].ref; + var searchitem = '
  • ' + store[ref].t + '
    • '; + $('.searchResultList').append(searchitem); + } + $('#searchResult').removeClass('hidden'); } - $('#searchResult').removeClass('hidden'); } }); }) diff --git a/routes/api.js b/routes/api.js index ff929f07..f6e6f989 100644 --- a/routes/api.js +++ b/routes/api.js @@ -60,7 +60,6 @@ router.post('/api/validate_permalink', function(req, res){ // public API for inserting posts router.post('/api/newArticle', function(req, res){ var db = req.app.db; - var lunr_index = req.lunr_index; var Validator = require('jsonschema').Validator; var v = new Validator(); @@ -149,11 +148,6 @@ router.post('/api/newArticle', function(req, res){ res.status(400).json({result: false, errors: [err]}); return; } - // setup keywords - var keywords = ''; - if(req.body.kb_keywords !== undefined){ - keywords = req.body.kb_keywords.toString().replace(/,/g, ' '); - } // get the new ID var newId = newDoc._id; @@ -161,26 +155,14 @@ router.post('/api/newArticle', function(req, res){ newId = newDoc.insertedIds; } - // create lunr doc - var lunr_doc = { - kb_title: req.body.kb_title, - kb_keywords: keywords, - id: newId - }; - - // if index body is switched on - if(config.settings.index_article_body === true){ - lunr_doc['kb_body'] = req.body.kb_body; - } - // add to store var href = req.body.kb_permalink !== '' ? req.body.kb_permalink : newId; - req.lunr_store[newId] = {t: req.body.kb_title, p: href}; - - // add to lunr index - lunr_index.add(lunr_doc); + common.updateStore(newId, {t: req.body.kb_title, p: href}); - res.status(200).json({result: true, message: 'All good'}); + // rebuild index + common.buildIndex(db, function(){ + res.status(200).json({result: true, message: 'All good'}); + }); }); }); }else{ diff --git a/routes/common.js b/routes/common.js index 1d231c5d..af5d0252 100644 --- a/routes/common.js +++ b/routes/common.js @@ -1,5 +1,6 @@ var path = require('path'); var fs = require('fs'); +var lunr = require('lunr'); exports.clear_session_value = function (session, session_var){ var temp = session[session_var]; @@ -7,7 +8,7 @@ exports.clear_session_value = function (session, session_var){ return temp; }; -exports.read_config = function (){ +exports.read_config = function(){ // preferred path var configFile = path.join(__dirname, '..', 'config', 'config.json'); @@ -41,6 +42,82 @@ exports.read_config = function (){ return loadedConfig; }; +exports.getIndex = function(){ + var index = {}; + if(fs.existsSync(path.join('data', 'searchIndex.json'))){ + index.index = fs.readFileSync(path.join('data', 'searchIndex.json'), 'utf8'); + }; + if(fs.existsSync(path.join('data', 'searchStore.json'))){ + index.store = fs.readFileSync(path.join('data', 'searchStore.json'), 'utf8'); + }; + + return index; +} + +exports.updateStore = function(id, doc){ + var lunr_store = this.getIndex().store; + lunr_store[id] = doc; + fs.writeFileSync(path.join('data', 'searchStore.json'), JSON.stringify(lunr_store), 'utf-8'); +} + +exports.removeStore = function(id){ + var lunr_store = this.getIndex().store; + delete lunr_store[id]; + fs.writeFileSync(path.join('data', 'searchStore.json'), JSON.stringify(lunr_store), 'utf-8'); +} + +exports.buildIndex = function(db, callback){ + var config = this.read_config(); + // get all articles on startup + db.kb.find({kb_published: 'true'}, function (err, kb_list){ + // build the index + var lunr_store = {}; + var idx = lunr(function(){ + this.field('kb_title'); + this.field('kb_keywords') + this.ref('id'); + + if(config.settings.index_article_body === true){ + this.field('kb_body'); + } + + var index = this; + + // add to lunr index + kb_list.forEach(function(kb){ + // only if defined + var keywords = ''; + if(kb.kb_keywords !== undefined){ + keywords = kb.kb_keywords.toString().replace(/,/g, ' '); + } + + var doc = { + 'kb_title': kb.kb_title, + 'kb_keywords': keywords, + 'id': kb._id + }; + + // if index body is switched on + if(config.settings.index_article_body === true){ + doc['kb_body'] = kb.kb_body; + } + + // add to store + var href = kb.kb_permalink !== '' ? kb.kb_permalink : kb._id; + lunr_store[kb._id] = {t: kb.kb_title, p: href}; + + index.add(doc); + }); + }); + + // write out our index + fs.writeFileSync(path.join('data', 'searchIndex.json'), JSON.stringify(idx), 'utf-8'); + fs.writeFileSync(path.join('data', 'searchStore.json'), JSON.stringify(lunr_store), 'utf-8'); + + callback(null); + }); +} + // This is called on the suggest url. If the value is set to false in the config // a 403 error is rendered. exports.suggest_allowed = function (req, res, next){ diff --git a/routes/index.js b/routes/index.js index 2492487e..5f1cb229 100644 --- a/routes/index.js +++ b/routes/index.js @@ -65,9 +65,8 @@ router.post('/protected/action', function (req, res){ }); router.post('/search_api', function (req, res){ - var lunr_index = req.lunr_index; - var lunr_store = req.lunr_store; - res.status(200).json({index: lunr_index, store: lunr_store}); + var index = common.getIndex(); + res.status(200).json(index); }); // vote on articles @@ -370,7 +369,6 @@ router.get('/edit/:id', common.restrict, function (req, res){ // insert new KB form action router.post('/insert_kb', common.restrict, function (req, res){ var db = req.app.db; - var lunr_index = req.lunr_index; var doc = { kb_permalink: req.body.frm_kb_permalink, @@ -428,30 +426,18 @@ router.post('/insert_kb', common.restrict, function (req, res){ newId = newDoc.insertedIds; } - // create lunr doc - var lunr_doc = { - kb_title: req.body.frm_kb_title, - kb_keywords: keywords, - id: newId - }; - - // if index body is switched on - if(config.settings.index_article_body === true){ - lunr_doc['kb_body'] = req.body.frm_kb_body; - } - // add to store var href = req.body.frm_kb_permalink !== '' ? req.body.frm_kb_permalink : newId; - req.lunr_store[newId] = {t: req.body.frm_kb_title, p: href}; + common.updateStore(newId, {t: req.body.frm_kb_title, p: href}); // add to lunr index - lunr_index.add(lunr_doc); - - req.session.message = req.i18n.__('New article successfully created'); - req.session.message_type = 'success'; + common.buildIndex(db, function(){ + req.session.message = req.i18n.__('New article successfully created'); + req.session.message_type = 'success'; - // redirect to new doc - res.redirect(req.app_context + '/edit/' + newId); + // redirect to new doc + res.redirect(req.app_context + '/edit/' + newId); + }); } }); } @@ -478,7 +464,6 @@ router.get('/suggest', common.suggest_allowed, function (req, res){ // Update an existing KB article form action router.post('/insert_suggest', common.suggest_allowed, function (req, res){ var db = req.app.db; - var lunr_index = req.lunr_index; // if empty, remove the comma and just have a blank string var keywords = req.body.frm_kb_keywords; @@ -502,41 +487,18 @@ router.post('/insert_suggest', common.suggest_allowed, function (req, res){ req.session.message_type = 'danger'; res.redirect(req.app_context + '/'); }else{ - // setup keywords - var keywords = ''; - if(req.body.frm_kb_keywords !== undefined){ - keywords = req.body.frm_kb_keywords.toString().replace(/,/g, ' '); - } - - // get the new ID - var newId = newDoc._id; - if(config.settings.database.type !== 'embedded'){ - newId = newDoc.insertedIds; - } - - // create lunr doc - var lunr_doc = { - kb_title: req.body.frm_kb_title, - kb_keywords: keywords, - id: newId - }; - - // if index body is switched on - if(config.settings.index_article_body === true){ - lunr_doc['kb_body'] = req.body.frm_kb_body; - } // update store var href = newId; - req.lunr_store[newId] = {t: req.body.frm_kb_title, p: href}; - - // add to lunr index - lunr_index.add(lunr_doc); + common.updateStore(newId, {t: req.body.frm_kb_title, p: href}) - // redirect to new doc - req.session.message = req.i18n.__('Suggestion successfully processed'); - req.session.message_type = 'success'; - res.redirect(req.app_context + '/'); + // rebuild index + common.buildIndex(db, function(){ + // redirect to new doc + req.session.message = req.i18n.__('Suggestion successfully processed'); + req.session.message_type = 'success'; + res.redirect(req.app_context + '/'); + }); } }); }); @@ -544,7 +506,6 @@ router.post('/insert_suggest', common.suggest_allowed, function (req, res){ // Update an existing KB article form action router.post('/save_kb', common.restrict, function (req, res){ var db = req.app.db; - var lunr_index = req.lunr_index; var kb_featured = req.body.frm_kb_featured === 'on' ? 'true' : 'false'; // if empty, remove the comma and just have a blank string @@ -616,61 +577,49 @@ router.post('/save_kb', common.restrict, function (req, res){ keywords = req.body.frm_kb_keywords.toString().replace(/,/g, ' '); } - // create lunr doc - var lunr_doc = { - kb_title: req.body.frm_kb_title, - kb_keywords: keywords, - id: req.body.frm_kb_id - }; - - // if index body is switched on - if(config.settings.index_article_body === true){ - lunr_doc['kb_body'] = req.body.frm_kb_body; - } - // update store var href = req.body.frm_kb_permalink !== '' ? req.body.frm_kb_permalink : req.body.frm_kb_id; - req.lunr_store[req.body.frm_kb_id] = {t: req.body.frm_kb_title, p: href}; - - // update the index - lunr_index.update(lunr_doc, false); - - var article_versioning = config.settings.article_versioning ? config.settings.article_versioning : false; - - // if versions turned on, insert a doc to track versioning - if(article_versioning === true){ - // version doc - var version_doc = { - kb_title: req.body.frm_kb_title, - kb_parent_id: req.body.frm_kb_id, - kb_versioned_doc: true, - kb_edit_reason: req.body.frm_kb_edit_reason, - kb_body: req.body.frm_kb_body, - kb_published: false, - kb_keywords: keywords, - kb_last_updated: new Date(), - kb_last_update_user: req.session.users_name + ' - ' + req.session.user, - kb_author: author, - kb_author_email: author_email, - kb_published_date: published_date, - kb_password: req.body.frm_kb_password, - kb_permalink: req.body.frm_kb_permalink, - kb_featured: kb_featured, - kb_seo_title: req.body.frm_kb_seo_title, - kb_seo_description: req.body.frm_kb_seo_description - }; - - // insert a doc to track versioning - db.kb.insert(version_doc, function (err, version_doc){ + common.updateStore(req.body.frm_kb_id, {t: req.body.frm_kb_title, p: href}); + + // rebuild the index + common.buildIndex(db, function(){ + var article_versioning = config.settings.article_versioning ? config.settings.article_versioning : false; + + // if versions turned on, insert a doc to track versioning + if(article_versioning === true){ + // version doc + var version_doc = { + kb_title: req.body.frm_kb_title, + kb_parent_id: req.body.frm_kb_id, + kb_versioned_doc: true, + kb_edit_reason: req.body.frm_kb_edit_reason, + kb_body: req.body.frm_kb_body, + kb_published: false, + kb_keywords: keywords, + kb_last_updated: new Date(), + kb_last_update_user: req.session.users_name + ' - ' + req.session.user, + kb_author: author, + kb_author_email: author_email, + kb_published_date: published_date, + kb_password: req.body.frm_kb_password, + kb_permalink: req.body.frm_kb_permalink, + kb_featured: kb_featured, + kb_seo_title: req.body.frm_kb_seo_title, + kb_seo_description: req.body.frm_kb_seo_description + }; + + // insert a doc to track versioning + db.kb.insert(version_doc, function (err, version_doc){ + req.session.message = req.i18n.__('Successfully saved'); + req.session.message_type = 'success'; + res.redirect(req.app_context + '/edit/' + req.body.frm_kb_id); + }); + }else{ req.session.message = req.i18n.__('Successfully saved'); req.session.message_type = 'success'; res.redirect(req.app_context + '/edit/' + req.body.frm_kb_id); - }); - }else{ - req.session.message = req.i18n.__('Successfully saved'); - req.session.message_type = 'success'; - res.redirect(req.app_context + '/edit/' + req.body.frm_kb_id); - } + } + }); } }); }); @@ -788,7 +737,7 @@ router.get('/articles/all', common.restrict, function (req, res){ router.get('/articles/:tag', function (req, res){ var db = req.app.db; - var lunr_index = req.lunr_index; + var lunr_index = common.getIndex().index; // we strip the ID's from the lunr index search var lunr_id_array = []; @@ -1088,7 +1037,6 @@ router.get('/user/delete/:id', common.restrict, function (req, res){ // delete article router.get('/delete/:id', common.restrict, function (req, res){ var db = req.app.db; - var lunr_index = req.lunr_index; // remove the article db.kb.remove({_id: common.getId(req.params.id)}, {}, function (err, numRemoved){ @@ -1104,15 +1052,15 @@ router.get('/delete/:id', common.restrict, function (req, res){ }; // remove from store - delete req.lunr_store[req.params.id]; - - // remove the index - lunr_index.remove(lunr_doc, false); + common.removeStore(req.params.id); - // redirect home - req.session.message = req.i18n.__('Article successfully deleted'); - req.session.message_type = 'success'; - res.redirect(req.app_context + '/articles'); + // remove from index + common.buildIndex(db, function(){ + // redirect home + req.session.message = req.i18n.__('Article successfully deleted'); + req.session.message_type = 'success'; + res.redirect(req.app_context + '/articles'); + }) }); }); @@ -1315,7 +1263,7 @@ router.get(['/search/:tag', '/topic/:tag'], common.restrict, function (req, res) var db = req.app.db; common.config_expose(req.app); var search_term = req.params.tag; - var lunr_index = req.lunr_index; + var lunr_index = common.getIndex().index; // determine whether its a search or a topic var routeType = 'search'; @@ -1368,7 +1316,7 @@ router.post('/search', common.restrict, function (req, res){ var db = req.app.db; common.config_expose(req.app); var search_term = req.body.frm_search; - var lunr_index = req.lunr_index; + var lunr_index = common.getIndex().index; // we strip the ID's from the lunr index search var lunr_id_array = []; @@ -1550,15 +1498,14 @@ router.get('/sitemap.xml', function (req, res, next){ } // create the sitemap - var sitemap = sm.createSitemap( - { - hostname: req.protocol + '://' + req.headers.host, - cacheTime: 600000, // 600 sec - cache purge period - urls: urlArray - }); + var sitemap = sm.createSitemap({ + hostname: req.protocol + '://' + req.headers.host, + cacheTime: 600000, // 600 sec - cache purge period + urls: urlArray + }); // render the sitemap - sitemap.toXML(function (err, xml){ + sitemap.toXML(function(err, xml){ if(err){ return res.status(500).end(); } diff --git a/views/layouts/layout.hbs b/views/layouts/layout.hbs index 072b3f9b..fcb78591 100644 --- a/views/layouts/layout.hbs +++ b/views/layouts/layout.hbs @@ -78,7 +78,7 @@ {{#ifCond config.settings.typeahead_search '===' true}} - + {{/ifCond}}