parser: remove the old lexer, clean up (pingcap#1597)

Makefile

@@ -35,7 +35,7 @@ LDFLAGS += -X "github.com/pingcap/tidb/util/printer.TiDBGitHash=$(shell git rev-
 
 TARGET = ""
 
-.PHONY: all build install update parser clean todo test gotest interpreter server goyacc golex dev
+.PHONY: all build install update parser clean todo test gotest interpreter server goyacc dev
 
 default: server
 
@@ -55,15 +55,12 @@ install:
 
 TEMP_FILE = temp_parser_file
 
-golex:
-	$(GO) get github.com/qiuyesuifeng/golex
-
 goyacc:
 	rm -rf vendor && ln -s _vendor/vendor vendor
 	$(GO) build -o bin/goyacc github.com/pingcap/tidb/parser/goyacc
 	rm -rf vendor
 
-parser: goyacc golex
+parser: goyacc
 	bin/goyacc -o /dev/null -xegen $(TEMP_FILE) parser/parser.y
 	bin/goyacc -o parser/parser.go -xe $(TEMP_FILE) parser/parser.y 2>&1 | egrep "(shift|reduce)/reduce" | awk '{print} END {if (NR > 0) {print "Find conflict in parser.y. Please check y.output for more information."; system("rm -f $(TEMP_FILE)"); exit 1;}}'
 	rm -f $(TEMP_FILE)
@@ -78,9 +75,7 @@ parser: goyacc golex
 		/usr/bin/sed -i "" 's/yyEofCode/yyEOFCode/' parser/parser.go; \
 	fi
 
-	$(GOLEX) -o parser/scanner.go parser/scanner.l
 	@awk 'BEGIN{print "// Code generated by goyacc"} {print $0}' parser/parser.go > tmp_parser.go && mv tmp_parser.go parser/parser.go;
-	@awk 'BEGIN{print "// Code generated by goyacc"} {print $0}' parser/scanner.go > tmp_scanner.go && mv tmp_scanner.go parser/scanner.go;
 
 check:
 	bash gitcookie.sh

parser/misc.go

@@ -15,7 +15,9 @@ package parser
 
 import (
 	"bytes"
+	"strings"
 
+	"github.com/pingcap/tidb/util/charset"
 	"github.com/pingcap/tidb/util/hack"
 )
 
@@ -455,3 +457,19 @@ func isTokenIdentifier(s string, buf *bytes.Buffer) int {
 	tok := tokenMap[hack.String(data)]
 	return tok
 }
+
+func handleIdent(lval *yySymType) int {
+	s := lval.ident
+	// A character string literal may have an optional character set introducer and COLLATE clause:
+	// [_charset_name]'string' [COLLATE collation_name]
+	// See https://dev.mysql.com/doc/refman/5.7/en/charset-literal.html
+	if !strings.HasPrefix(s, "_") {
+		return identifier
+	}
+	cs, _, err := charset.GetCharsetInfo(s[1:])
+	if err != nil {
+		return identifier
+	}
+	lval.item = cs
+	return underscoreCS
+}

parser/parser_test.go

@@ -15,7 +15,6 @@ package parser
 
 import (
 	"fmt"
-	"reflect"
 	"runtime"
 	"strings"
 	"testing"
@@ -1024,50 +1023,3 @@ func BenchmarkParse(b *testing.B) {
 	}
 	b.ReportAllocs()
 }
-
-var tableCompatible = []string{
-	`drop table IF EXISTS t;
-	CREATE TABLE t(c INT, index cidx (c));`,
-	`INSERT INTO t VALUES(1), (null), (2);`,
-	"SELECT COUNT(c) FROM t WHERE c IS NOT NULL;",
-	`select cast(null as char(30))`,
-	`select 0b01 + 1, 0b01000001 = "A"`,
-	"create table t (id tiny)",
-	"select        ((a+1))     from t",
-	"select !true from t",
-	"select IF(1>2,2,3) from t",
-	"select hex('TiDB') from t",
-	`show tables like 'show\_test'`,
-	`select _utf8"string";`,
-	"alter table show_test drop foreign key `fk`",
-	`insert t values('\x01')`,
-	`select "ab\_c"`,
-	`select "ab\%c"`,
-	`SELECT CONVERT("ABCD" USING ASCII);`,
-}
-
-func (s *testParserSuite) TestParserCompatible(c *C) {
-	defer testleak.AfterTest(c)()
-	saveUseNewLexer := UseNewLexer
-
-	parser := New()
-	for _, str := range tableCompatible {
-		st1, err1 := parser.Parse(str, "", "")
-
-		UseNewLexer = true
-		st2, err2 := parser.Parse(str, "", "")
-		UseNewLexer = false
-
-		c.Assert(err1, IsNil)
-		c.Assert(err2, IsNil)
-		c.Assert(len(st1), Equals, len(st2))
-
-		for i := 0; i < len(st1); i++ {
-			if !reflect.DeepEqual(st1[i], st2[i]) {
-				c.Error(i, st1[i], st2[i])
-			}
-		}
-	}
-
-	UseNewLexer = saveUseNewLexer
-}