Fixed bug where multiple identical parses were created for an input

in an unambiguous grammar in some situation involving nullable nonterminals
(issue kach#85).

.gitignore

@@ -3,6 +3,7 @@ node_modules/*
 .DS_Store
 test/profile.log
 test/parens.js
+test/whitespace.js
 *.swp
 *.ps1
 *.cmd

lib/nearley.js

@@ -62,7 +62,7 @@ State.prototype.consumeNonTerminal = function(inp) {
  return false;
 };
 
-State.prototype.process = function(location, table, rules, addedRules) {
+State.prototype.process = function(location, ind, table, rules, addedRules) {
  if (this.expect === this.rule.symbols.length) {
  // I have completed a rule
  if (this.rule.postprocess) {
@@ -78,7 +78,7 @@ State.prototype.process = function(location, table, rules, addedRules) {
  x = s.consumeNonTerminal(this.rule.name);
  if (x) {
  x.data[x.data.length-1] = this.data;
- table[location].push(x);
+ push_epsilon(location, ind, table, x);
  }
  w++;
  }
@@ -101,21 +101,6 @@ State.prototype.process = function(location, table, rules, addedRules) {
  // }
  }
  } else {
- // In case I missed an older nullable's sweep, update yourself. See
- // above context for why this makes sense.
-
- var ind = table[location].indexOf(this);
- for (var i=0; i<ind; i++) {
- var state = table[location][i];
- if (state.rule.symbols.length === state.expect && state.reference === location) {
- var x = this.consumeNonTerminal(state.rule.name);
- if (x) {
- x.data[x.data.length-1] = state.data;
- table[location].push(x);
- }
- }
- }
-
  // I'm not done, but I can predict something
  var exp = this.rule.symbols[this.expect];
 
@@ -137,7 +122,7 @@ State.prototype.process = function(location, table, rules, addedRules) {
 
  if (r.symbols.length > 0) {
  addedRules.push(r);
- table[location].push(new State(r, 0, location));
+ push_epsilon(location, ind, table, new State(r, 0, location));
  } else {
  // Empty rule
  // This is special
@@ -147,13 +132,38 @@ State.prototype.process = function(location, table, rules, addedRules) {
  } else {
  copy.data[copy.data.length-1] = [];
  }
- table[location].push(copy);
+ push_epsilon(location, ind, table, copy);
  }
  }
  });
  }
 };
 
+State.prototype.isComplete = function() {
+ return this.expect === this.rule.symbols.length;
+}
+
+
+function push_epsilon(location, ind, table, newState) {
+ var col = table[location];
+
+ col.push(newState);
+
+ if (!newState.isComplete()) {
+ // In case I missed an older nullable's sweep, update yourself. See
+ // State.prototype.process for why this makes sense.
+ for (var i=0; i<ind; i++) {
+ var state = col[i];
+ if (state.isComplete() && state.reference === location) {
+ var x = newState.consumeNonTerminal(state.rule.name);
+ if (x) {
+ x.data[x.data.length-1] = state.data;
+ push_epsilon(location, ind, table, x);
+ }
+ }
+ }
+ }
+}
 
 
 function Parser(rules, start) {
@@ -180,7 +190,7 @@ Parser.prototype.advanceTo = function(n, addedRules) {
  // Advance a table, take the closure of .process for location n in the input stream
  var w = 0;
  while (w < this.table[n].length) {
- (this.table[n][w]).process(n, this.table, this.rules, addedRules);
+ (this.table[n][w]).process(n, w, this.table, this.rules, addedRules);
  w++;
  }
 }

test/launch.sh

@@ -14,13 +14,18 @@ time bin/nearleyc.js test/indentation.ne > /dev/null
 echo "Testing percent bug..."
 bin/nearleyc.js test/percent.ne > /dev/null
 
+echo "Testing nullable whitespace bug..."
+bin/nearleyc.js test/whitespace.ne -o test/whitespace.js
+bin/nearley-test.js test/whitespace.js -i "(x)" | sed -e '1,/Parse results:/d' \
+ | diff - test/whitespace.expected || echo "[ FAILED ]"
+
 if ! type "coffee" > /dev/null; then
  echo "Coffeescript not installed, skipping coffeescript test!";
 else
  echo "Testing coffeescript template";
  bin/nearleyc.js test/coffeescript-test.ne -o test/tmp.coffeescript-test.coffee;
  coffee -c test/tmp.coffeescript-test.coffee;
- nearley-test test/tmp.coffeescript-test.js -i "ABCDEFZ12309" > /dev/null;
+ bin/nearley-test.js test/tmp.coffeescript-test.js -i "ABCDEFZ12309" > /dev/null;
 fi
 
 echo "Done with all tests."

test/whitespace.expected

@@ -0,0 +1 @@
+[ [ [ [ '(', null, [ [ [ [ 'x' ] ] ] ], null, ')' ] ] ] ]

test/whitespace.ne

@@ -0,0 +1,11 @@
+@builtin "whitespace.ne"
+
+d -> a
+
+a -> b _ "&"
+ | b
+
+b -> letter
+ | "(" _ d _ ")"
+
+letter -> [a-z]