Fix mrvautin#171 - Moving to a stored Lunr index and upgrading to lat…

…est version (2.0.4).
mindis · May 25, 2017 · 8d1bc71 · 8d1bc71
1 parent 9d468e6
commit 8d1bc71
Show file tree

Hide file tree

Showing 9 changed files with 180 additions and 250 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -2,6 +2,6 @@ language: node_js
 node_js:
   - "node"
   - "iojs"
-  - "0.12"
-  - "0.11"
-  - "0.10"
+  - "6"
+  - "4"
+  - "0.12"
diff --git a/README.md b/README.md
@@ -19,6 +19,8 @@ Demo: [http://openkb.markmoffat.com](http://openkb.markmoffat.com)
 3. Start application: `npm start`
 4. Go to  [http://127.0.0.1:4444](http://127.0.0.1:4444) in your browser
 
+> Note: `openKB` supports Nodejs version 0.12 and above.
+
 ### Deploy on Heroku
 
 [![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy?template=https://github.com/mrvautin/openKB)

diff --git a/app.js b/app.js
@@ -7,7 +7,6 @@ var bodyParser = require('body-parser');
 var Nedb = require('nedb');
 var session = require('express-session');
 var bcrypt = require('bcrypt-nodejs');
-var lunr = require('lunr');
 var markdownit = require('markdown-it')({html: true, linkify: true, typographer: true});
 var moment = require('moment');
 var fs = require('fs');
@@ -18,7 +17,6 @@ var config = common.read_config();
 var MongoClient = require('mongodb').MongoClient;
 var expstate = require('express-state');
 var compression = require('compression');
-var lunr_store = {};
 
 // require the routes
 var index = require('./routes/index');
@@ -224,8 +222,6 @@ app.use(function (req, res, next){
 	req.handlebars = handlebars.helpers;
     req.bcrypt = bcrypt;
     req.i18n = i18n;
-    req.lunr_index = lunr_index;
-    req.lunr_store = lunr_store;
     req.app_context = app_context;
     req.i18n.setLocaleFromCookie();
 	next();
@@ -290,7 +286,7 @@ if(config.settings.database.type === 'embedded'){
     app.db = db;
 
     // add articles to index
-    indexArticles(db, function(err){
+    common.buildIndex(db, function(err){
         // lift the app
         app.listen(app.get('port'), app.get('bind'), function (){
             console.log('openKB running on host: http://' + app.get('bind') + ':' + app.get('port'));
@@ -314,7 +310,7 @@ if(config.settings.database.type === 'embedded'){
         app.db = db;
 
         // add articles to index
-        indexArticles(db, function(err){
+        common.buildIndex(db, function(err){
             // lift the app
             app.listen(app.get('port'), app.get('bind'), function (){
                 console.log('openKB running on host: http://' + app.get('bind') + ':' + app.get('port'));
@@ -324,80 +320,4 @@ if(config.settings.database.type === 'embedded'){
     });
 }
 
-// setup lunr indexing
-var lunr_index = lunr(function (){
-    this.field('kb_title', {boost: 10});
-    this.field('kb_keywords');
-});
-
-// if index body is switched on
-if(config.settings.index_article_body === true){
-    lunr_index.field('kb_body');
-}
-
-function indexArticles(db, callback){
-    // get all articles on startup
-    if(config.settings.database.type === 'embedded'){
-        db.kb.find({kb_published: 'true'}, function (err, kb_list){
-            // add to lunr index
-            kb_list.forEach(function(kb){
-                // only if defined
-                var keywords = '';
-                if(kb.kb_keywords !== undefined){
-                    keywords = kb.kb_keywords.toString().replace(/,/g, ' ');
-                }
-
-                var doc = {
-                    'kb_title': kb.kb_title,
-                    'kb_keywords': keywords,
-                    'id': kb._id
-                };
-
-                // if index body is switched on
-                if(config.settings.index_article_body === true){
-                    doc['kb_body'] = kb.kb_body;
-                }
-
-                // add to store
-                var href = kb.kb_permalink !== '' ? kb.kb_permalink : kb._id;
-                lunr_store[kb._id] = {t: kb.kb_title, p: href};
-
-                lunr_index.add(doc);
-            });
-
-            callback(null);
-        });
-    }else{
-        db.kb.find({kb_published: 'true'}).toArray(function (err, kb_list){
-            // add to lunr index
-            kb_list.forEach(function(kb){
-                // only if defined
-                var keywords = '';
-                if(typeof kb.kb_keywords !== 'undefined' && kb.kb_keywords !== null){
-                    keywords = kb.kb_keywords.toString().replace(/,/g, ' ');
-                }
-
-                var doc = {
-                    'kb_title': kb.kb_title,
-                    'kb_keywords': keywords,
-                    'id': kb._id
-                };
-
-                // if index body is switched on
-                if(config.settings.index_article_body === true){
-                    doc['kb_body'] = kb.kb_body;
-                }
-
-                // add to store
-                var href = kb.kb_permalink !== '' ? kb.kb_permalink : kb._id;
-                lunr_store[kb._id] = {t: kb.kb_title, p: href};
-
-                lunr_index.add(doc);
-            });
-
-            callback(null);
-        });
-    }
-}
-
 module.exports = app;
diff --git a/package.json b/package.json
@@ -29,7 +29,7 @@
     "jszip": "^2.6.1",
     "junk": "^2.0.0",
     "lodash": "^4.17.2",
-    "lunr": "^0.7.1",
+    "lunr": "^2.0.4",
     "markdown-it": "^7.0.1",
     "mime-types": "^2.1.13",
     "mkdirp": "^0.5.1",
@@ -72,5 +72,5 @@
   "bugs": {
     "url": "https://github.com/mrvautin/openKB/issues"
   },
-  "homepage": "http://openkb.mrvautin.com"
+  "homepage": "https://openkb.markmoffat.com"
 }
diff --git a/public/javascripts/openKB.js b/public/javascripts/openKB.js
@@ -48,23 +48,25 @@ $(document).ready(function(){
                 data: {id: this.id, state: this.checked}
             })
             .done(function(response){
-                var index = lunr.Index.load(response.index);
-                var store = response.store;
+                var index = lunr.Index.load(JSON.parse(response.index));
+                var store = JSON.parse(response.store);
 
                 $('#frm_search').on('keyup', function(){
                     var query = $(this).val();
-                    var results = index.search(query);
-                    if(results.length === 0){
-                        $('#searchResult').addClass('hidden');
-                    }else{
-                        $('.searchResultList').empty();
-                        $('.searchResultList').append('<li class="list-group-item list-group-heading">Search results</li>');
-                        for(var result in results){
-                            var ref = results[result].ref;
-                            var searchitem = '<li class="list-group-item"><a href="' + $('#app_context').val() + '/' + config.route_name + '/' + store[ref].p + '">' + store[ref].t + '</a></li>';
-                            $('.searchResultList').append(searchitem);
+                    if(query.length > 2){
+                        var results = index.search(query);
+                        if(results.length === 0){
+                            $('#searchResult').addClass('hidden');
+                        }else{
+                            $('.searchResultList').empty();
+                            $('.searchResultList').append('<li class="list-group-item list-group-heading">Search results</li>');
+                            for(var result in results){
+                                var ref = results[result].ref;
+                                var searchitem = '<li class="list-group-item"><a href="' + $('#app_context').val() + '/' + config.route_name + '/' + store[ref].p + '">' + store[ref].t + '</a></li>';
+                                $('.searchResultList').append(searchitem);
+                            }
+                            $('#searchResult').removeClass('hidden');
                         }
-                        $('#searchResult').removeClass('hidden');
                     }
                 });
             })

diff --git a/routes/api.js b/routes/api.js
@@ -60,7 +60,6 @@ router.post('/api/validate_permalink', function(req, res){
 // public API for inserting posts
 router.post('/api/newArticle', function(req, res){
     var db = req.app.db;
-    var lunr_index = req.lunr_index;
     var Validator = require('jsonschema').Validator;
     var v = new Validator();
 
@@ -149,38 +148,21 @@ router.post('/api/newArticle', function(req, res){
                                 res.status(400).json({result: false, errors: [err]});
                                 return;
                             }
-                            // setup keywords
-                            var keywords = '';
-                            if(req.body.kb_keywords !== undefined){
-                                keywords = req.body.kb_keywords.toString().replace(/,/g, ' ');
-                            }
 
                             // get the new ID
                             var newId = newDoc._id;
                             if(config.settings.database.type !== 'embedded'){
                                 newId = newDoc.insertedIds;
                             }
 
-                            // create lunr doc
-                            var lunr_doc = {
-                                kb_title: req.body.kb_title,
-                                kb_keywords: keywords,
-                                id: newId
-                            };
-
-                            // if index body is switched on
-                            if(config.settings.index_article_body === true){
-                                lunr_doc['kb_body'] = req.body.kb_body;
-                            }
-
                             // add to store
                             var href = req.body.kb_permalink !== '' ? req.body.kb_permalink : newId;
-                            req.lunr_store[newId] = {t: req.body.kb_title, p: href};
-
-                            // add to lunr index
-                            lunr_index.add(lunr_doc);
+                            common.updateStore(newId, {t: req.body.kb_title, p: href});
 
-                            res.status(200).json({result: true, message: 'All good'});
+                            // rebuild index
+                            common.buildIndex(db, function(){
+                                res.status(200).json({result: true, message: 'All good'});
+                            });
                         });
                     });
                 }else{

diff --git a/routes/common.js b/routes/common.js
@@ -1,13 +1,14 @@
 var path = require('path');
 var fs = require('fs');
+var lunr = require('lunr');
 
 exports.clear_session_value = function (session, session_var){
     var temp = session[session_var];
     session[session_var] = null;
     return temp;
 };
 
-exports.read_config = function (){
+exports.read_config = function(){
     // preferred path
     var configFile = path.join(__dirname, '..', 'config', 'config.json');
 
@@ -41,6 +42,82 @@ exports.read_config = function (){
     return loadedConfig;
 };
 
+exports.getIndex = function(){
+    var index = {};
+    if(fs.existsSync(path.join('data', 'searchIndex.json'))){
+        index.index = fs.readFileSync(path.join('data', 'searchIndex.json'), 'utf8');
+    };
+    if(fs.existsSync(path.join('data', 'searchStore.json'))){
+        index.store = fs.readFileSync(path.join('data', 'searchStore.json'), 'utf8');
+    };
+
+    return index;
+}
+
+exports.updateStore = function(id, doc){
+    var lunr_store = this.getIndex().store;
+    lunr_store[id] = doc;
+    fs.writeFileSync(path.join('data', 'searchStore.json'), JSON.stringify(lunr_store), 'utf-8');
+}
+
+exports.removeStore = function(id){
+    var lunr_store = this.getIndex().store;
+    delete lunr_store[id];
+    fs.writeFileSync(path.join('data', 'searchStore.json'), JSON.stringify(lunr_store), 'utf-8');
+}
+
+exports.buildIndex = function(db, callback){
+    var config = this.read_config();
+    // get all articles on startup
+    db.kb.find({kb_published: 'true'}, function (err, kb_list){
+        // build the index
+        var lunr_store = {};
+        var idx = lunr(function(){
+            this.field('kb_title');
+            this.field('kb_keywords')
+            this.ref('id');
+
+            if(config.settings.index_article_body === true){
+                this.field('kb_body');
+            }
+
+            var index = this;
+
+            // add to lunr index
+            kb_list.forEach(function(kb){
+                // only if defined
+                var keywords = '';
+                if(kb.kb_keywords !== undefined){
+                    keywords = kb.kb_keywords.toString().replace(/,/g, ' ');
+                }
+
+                var doc = {
+                    'kb_title': kb.kb_title,
+                    'kb_keywords': keywords,
+                    'id': kb._id
+                };
+
+                // if index body is switched on
+                if(config.settings.index_article_body === true){
+                    doc['kb_body'] = kb.kb_body;
+                }
+
+                // add to store
+                var href = kb.kb_permalink !== '' ? kb.kb_permalink : kb._id;
+                lunr_store[kb._id] = {t: kb.kb_title, p: href};
+
+                index.add(doc);
+            });
+        });
+
+        // write out our index
+        fs.writeFileSync(path.join('data', 'searchIndex.json'), JSON.stringify(idx), 'utf-8');
+        fs.writeFileSync(path.join('data', 'searchStore.json'), JSON.stringify(lunr_store), 'utf-8');
+
+        callback(null);
+    });
+}
+
 // This is called on the suggest url. If the value is set to false in the config
 // a 403 error is rendered.
 exports.suggest_allowed = function (req, res, next){
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,6 +2,6 @@ language: node_js @@
     node_js:
       - "node"
       - "iojs"
-      - "0.12"
-      - "0.11"
-      - "0.10"
+      - "6"
+      - "4"
+      - "0.12"