[javascript] problem with continue statement and newline followed by …

…whitespace (antlr#4046) * [javascript] problem with continue statement and newline followed by whitespace OR comment antlr#3772 * remove the here method because is not used anymore; * fix for the go base parser, lineTerminatorAhead should return false if the token is not on the hidden channel
bpurushotham9233 · Apr 15, 2024 · a8ba792 · a8ba792
1 parent dcfbe51
commit a8ba792
Show file tree

Hide file tree

Showing 8 changed files with 31 additions and 147 deletions.
diff --git a/javascript/javascript/CSharp/JavaScriptParserBase.cs b/javascript/javascript/CSharp/JavaScriptParserBase.cs
@@ -49,7 +49,7 @@ protected bool next(string str)
 
     protected bool notLineTerminator()
     {
-        return !here(LineTerminator);
+        return !lineTerminatorAhead();
     }
 
     protected bool notOpenBraceAndNotFunction()
@@ -63,30 +63,6 @@ protected bool closeBrace()
         return ((ITokenStream)this.InputStream).LT(1).Type == CloseBrace;
     }
 
-    /// <summary>Returns true if on the current index of the parser's
-    /// token stream a token of the given type exists on the
-    /// Hidden channel.
-    /// </summary>
-    /// <param name="type">
-    /// The type of the token on the Hidden channel to check.
-    /// </param>
-    protected bool here(int type)
-    {
-        // Get the most recently emitted token.
-        IToken currentToken = ((ITokenStream)this.InputStream).LT(-1);
-
-        // Get the next token index.
-        int nextTokenIndex = currentToken == null ? 0 : currentToken.TokenIndex + 1;
-
-        // Get the token after the `currentToken`. By using `_input.get(index)`,
-        // we also grab a token that is (possibly) on the HIDDEN channel.
-        IToken nextToken = ((ITokenStream)this.InputStream).Get(nextTokenIndex);
-
-        // Check if the token resides on the HIDDEN channel and if it's of the
-        // provided type.
-        return (nextToken.Channel == Lexer.Hidden) && (nextToken.Type == type);
-    }
-
     /// <summary>
     /// Returns true if on the current index of the parser's
     /// token stream a token exists on the Hidden channel which

diff --git a/javascript/javascript/Cpp/JavaScriptParserBase.cpp b/javascript/javascript/Cpp/JavaScriptParserBase.cpp
@@ -24,7 +24,7 @@ bool JavaScriptParserBase::next(std::string str)
 
 bool JavaScriptParserBase::notLineTerminator()
 {
-    return !here(JavaScriptParser::LineTerminator);
+    return !lineTerminatorAhead();
 }
 
 bool JavaScriptParserBase::notOpenBraceAndNotFunction()
@@ -39,23 +39,6 @@ bool JavaScriptParserBase::closeBrace()
     return _input->LT(1)->getType() == JavaScriptParser::CloseBrace;
 }
 
-bool JavaScriptParserBase::here(int type)
-{
-    // Get the most recently emitted token.
-    auto currentToken = _input->LT(-1);
-
-    // Get the next token index.
-    int nextTokenIndex = currentToken == nullptr ? 0 : currentToken->getTokenIndex() + 1;
-
-    // Get the token after the `currentToken`. By using `_input.get(index)`,
-    // we also grab a token that is (possibly) on the HIDDEN channel.
-    auto nextToken = _input->get(nextTokenIndex);
-
-    // Check if the token resides on the HIDDEN channel and if it's of the
-    // provided type.
-    return (nextToken->getChannel() == Lexer::HIDDEN) && (nextToken->getType() == type);
-}
-
 bool JavaScriptParserBase::lineTerminatorAhead()
 {
     // Get the token ahead of the current index.

diff --git a/javascript/javascript/Cpp/JavaScriptParserBase.h b/javascript/javascript/Cpp/JavaScriptParserBase.h
@@ -12,6 +12,5 @@ class JavaScriptParserBase : public antlr4::Parser {
     bool notLineTerminator();
     bool notOpenBraceAndNotFunction();
     bool closeBrace();
-    bool here(int type);
     bool lineTerminatorAhead();
 };
diff --git a/javascript/javascript/Go/javascript_parser_base.go b/javascript/javascript/Go/javascript_parser_base.go
@@ -32,7 +32,7 @@ func (p *JavaScriptParserBase) next(str string) bool {
 }
 
 func (p *JavaScriptParserBase) notLineTerminator() bool {
-	return !p.here(JavaScriptParserLineTerminator)
+	return !p.lineTerminatorAhead()
 }
 
 func (p *JavaScriptParserBase) notOpenBraceAndNotFunction() bool {
@@ -44,29 +44,6 @@ func (p *JavaScriptParserBase) closeBrace() bool {
 	return p.GetTokenStream().LT(1).GetTokenType() == JavaScriptParserCloseBrace
 }
 
-// Returns true if on the current index of the parser's
-// token stream a token of the given type exists on the
-// Hidden channel.
-func (p *JavaScriptParserBase) here(_type int) bool {
-	// Get the most recently emitted token.
-	currentToken := p.GetTokenStream().LT(-1)
-
-	// Get the next token index.
-	nextTokenIndex := 0
-
-	if currentToken != nil {
-		nextTokenIndex = currentToken.GetTokenIndex() + 1
-	}
-
-	// Get the token after the `currentToken`. By using `_input.get(index)`,
-	// we also grab a token that is (possibly) on the HIDDEN channel.
-	nextToken := p.GetTokenStream().Get(nextTokenIndex);
-
-	// Check if the token resides on the HIDDEN channel and if it's of the
-	// provided type.
-	return nextToken.GetChannel() == antlr.LexerHidden && nextToken.GetTokenType() == _type
-}
-
 // Returns true if on the current index of the parser's
 // token stream a token exists on the Hidden channel which
 // either is a line terminator, or is a multi line comment that
@@ -81,7 +58,7 @@ func (p *JavaScriptParserBase) lineTerminatorAhead() bool {
 
 	if ahead.GetChannel() != antlr.LexerHidden {
 		// We're only interested in tokens on the HIDDEN channel.
-		return true
+		return false
 	}
 
 	if ahead.GetTokenType() == JavaScriptParserLineTerminator {

diff --git a/javascript/javascript/Java/JavaScriptParserBase.java b/javascript/javascript/Java/JavaScriptParserBase.java
@@ -39,7 +39,7 @@ protected boolean next(String str) {
     }
 
     protected boolean notLineTerminator() {
-        return !here(JavaScriptParser.LineTerminator);
+        return !lineTerminatorAhead();
     }
 
     protected boolean notOpenBraceAndNotFunction() {
@@ -51,36 +51,6 @@ protected boolean closeBrace() {
         return _input.LT(1).getType() == JavaScriptParser.CloseBrace;
     }
 
-    /**
-     * Returns {@code true} iff on the current index of the parser's
-     * token stream a token of the given {@code type} exists on the
-     * {@code HIDDEN} channel.
-     *
-     * @param type
-     *         the type of the token on the {@code HIDDEN} channel
-     *         to check.
-     *
-     * @return {@code true} iff on the current index of the parser's
-     * token stream a token of the given {@code type} exists on the
-     * {@code HIDDEN} channel.
-     */
-    private boolean here(final int type) {
-
-        // Get the most recently emitted token.
-        Token currentToken = _input.LT(-1);
-
-        // Get the next token index.
-        int nextTokenIndex = currentToken == null ? 0 : currentToken.getTokenIndex() + 1;
-
-        // Get the token after the `currentToken`. By using `_input.get(index)`,
-        // we also grab a token that is (possibly) on the HIDDEN channel.
-        Token nextToken = _input.get(nextTokenIndex);
-
-        // Check if the token resides on the HIDDEN channel and if it's of the
-        // provided type.
-        return (nextToken.getChannel() == Lexer.HIDDEN) && (nextToken.getType() == type);
-    }
-
     /**
      * Returns {@code true} iff on the current index of the parser's
      * token stream a token exists on the {@code HIDDEN} channel which

diff --git a/javascript/javascript/JavaScript/JavaScriptParserBase.js b/javascript/javascript/JavaScript/JavaScriptParserBase.js
@@ -28,7 +28,7 @@ export default class JavaScriptParserBase extends antlr4.Parser {
     }
 
     notLineTerminator() {
-        return !this.here(JavaScriptParser.LineTerminator);
+        return !this.lineTerminatorAhead();
     }
 
     notOpenBraceAndNotFunction() {
@@ -43,22 +43,6 @@ export default class JavaScriptParserBase extends antlr4.Parser {
         return this._input.LT(1).type === JavaScriptParser.CloseBrace;
     }
 
-    here(type) {
-        // Get the most recently emitted token.
-        const currentToken = this._input.LT(-1);
-
-        // Get the next token index.
-        const nextTokenIndex = currentToken == null ? 0 : currentToken.tokenIndex + 1;
-
-        // Get the token after the `currentToken`. By using `_input.get(index)`,
-        // we also grab a token that is (possibly) on the HIDDEN channel.
-        const nextToken = this._input.get(nextTokenIndex);
-
-        // Check if the token resides on the HIDDEN channel and if it's of the
-        // provided type.
-        return nextToken.channel === antlr4.Lexer.HIDDEN && nextToken.type === type;
-    }
-
     lineTerminatorAhead() {
         let possibleIndexEosToken = this.getCurrentToken().tokenIndex - 1;
         if (possibleIndexEosToken < 0) return false;

diff --git a/javascript/javascript/Python3/JavaScriptParserBase.py b/javascript/javascript/Python3/JavaScriptParserBase.py
@@ -29,7 +29,7 @@ def next(self, s: str) -> bool:
     def notLineTerminator(self) -> bool:
         JavaScriptParser = self.parser()
 
-        return not self.here(JavaScriptParser.LineTerminator)
+        return not self.lineTerminatorAhead()
 
     def notOpenBraceAndNotFunction(self) -> bool:
         JavaScriptParser = self.parser()
@@ -42,35 +42,6 @@ def closeBrace(self) -> bool:
 
         return self._input.LT(1).type == JavaScriptParser.CloseBrace
 
-    def here(self, tokenType: int) -> bool:
-        """
-        Returns {@code true} iff on the current index of the parser's
-        token stream a token of the given {@code type} exists on the
-        {@code HIDDEN} channel.
-        :param:type:
-                   the type of the token on the {@code HIDDEN} channel
-                   to check.
-        :return:{@code true} iff on the current index of the parser's
-            token stream a token of the given {@code type} exists on the
-            {@code HIDDEN} channel.
-        """
-        # Get the token ahead of the current index.
-        assert isinstance(self.getCurrentToken(), Token)
-
-        # Get the most recently emitted token.
-        currentToken: Token = self._input.LT(-1)
-
-        # Get the next token index.
-        nextTokenIndex = 0 if currentToken is None else currentToken.tokenIndex + 1
-
-        # Get the token after the `currentToken`. By using `_input.get(index)`,
-        # we also grab a token that is (possibly) on the HIDDEN channel.
-        nextToken: Token = self._input.get(nextTokenIndex)
-
-        # Check if the token resides on the HIDDEN channel and if it's of the
-        # provided type.
-        return (nextToken.channel == Lexer.HIDDEN) and (nextToken.type == tokenType)
-
     def lineTerminatorAhead(self) -> bool:
         """
         Returns {@code true} iff on the current index of the parser's

diff --git a/javascript/javascript/examples/Continue.js b/javascript/javascript/examples/Continue.js
@@ -3,4 +3,28 @@ for (var i = 0; i < [].length; i++) {
     var o = {}
     if (false) continue
     o[test]
+
+    if (false) continue// comment
+    o[test]
+
+    if (false) continue // comment
+    o[test]
+
+    if (false) continue // comment
+
+    o[test]
+
+    if (false) continue/*
+         				* multiline comment
+         				*/
+    o[test]
+
+    if (false) continue /*
+         				 * multiline comment
+         				*/
+    o[test]
+}
+
+mylabel: for (var i = 0; i < [].length; i++) {	
+    if (true) continue mylabel;
 }