Add some documentation

* add documentation and golint

* merge with upstream/master and use gometaliner

* format code using make fmt

* remove some obvious comments from ast.go

* Clean up comments

* Clean up backend.go too

Co-authored-by: pallav.jha <[email protected]>
Co-authored-by: Phil Eaton <[email protected]>

Makefile

@@ -4,7 +4,7 @@ fmt:
 test:
 	go test -race -cover -coverprofile=coverage.out .
 
-cover: 
+cover:
 	go tool cover -func=coverage.out
 
 lint:

ast.go

@@ -49,17 +49,13 @@ func (e expression) generateCode() string {
 
 type selectItem struct {
 	exp      *expression
-	asterisk bool
+	asterisk bool // for *
 	as       *token
 }
 
-type fromItem struct {
-	table *token
-}
-
 type SelectStatement struct {
 	item  *[]*selectItem
-	from  *fromItem
+	from  *token
 	where *expression
 }
 
@@ -79,7 +75,7 @@ func (ss SelectStatement) GenerateCode() string {
 
 	from := ""
 	if ss.from != nil {
-		from = fmt.Sprintf("\nFROM\n\t\"%s\"", ss.from.table.value)
+		from = fmt.Sprintf("\nFROM\n\t\"%s\"", ss.from.value)
 	}
 
 	where := ""

ast_test.go

@@ -83,7 +83,7 @@ WHERE
 						{exp: &expression{literal: &token{value: "id", kind: identifierKind}, kind: literalKind}},
 						{exp: &expression{literal: &token{value: "name", kind: identifierKind}, kind: literalKind}},
 					},
-					from: &fromItem{&token{value: "users"}},
+					from: &token{value: "users"},
 					where: &expression{
 						binary: &binaryExpression{
 							a:  expression{literal: &token{value: "id", kind: identifierKind}, kind: literalKind},

lexer.go

@@ -5,11 +5,13 @@ import (
 	"strings"
 )
 
+// location of the token in source code
 type location struct {
 	line uint
 	col  uint
 }
 
+// for storing SQL reserved keywords
 type keyword string
 
 const (
@@ -37,6 +39,7 @@ const (
 	nullKeyword       keyword = "null"
 )
 
+// for storing SQL syntax
 type symbol string
 
 const (
@@ -111,17 +114,19 @@ func (t token) bindingPower() uint {
 	return 0
 }
 
-type cursor struct {
-	pointer uint
-	loc     location
-}
-
 func (t *token) equals(other *token) bool {
 	return t.value == other.value && t.kind == other.kind
 }
 
-type lexer func(string, cursor) (*token, cursor, bool)
+// cursor indicates the current position of the lexer
+type cursor struct {
+	pointer uint
+	loc     location
+}
 
+// longestMatch iterates through a source string starting at the given
+// cursor to find the longest matching substring among the provided
+// options
 func longestMatch(source string, ic cursor, options []string) string {
 	var value []byte
 	var skipList []int
@@ -339,7 +344,6 @@ func lexNumeric(source string, ic cursor) (*token, cursor, bool) {
 				cur.pointer++
 				cur.loc.col++
 			}
-
 			continue
 		}
 
@@ -360,6 +364,9 @@ func lexNumeric(source string, ic cursor) (*token, cursor, bool) {
 	}, cur, true
 }
 
+// lexCharacterDelimited looks through a source string starting at the
+// given cursor to find a start- and end- delimiter. The delimiter can
+// be escaped be preceeding the delimiter with itself.
 func lexCharacterDelimited(source string, ic cursor, delimiter byte) (*token, cursor, bool) {
 	cur := ic
 
@@ -388,11 +395,10 @@ func lexCharacterDelimited(source string, ic cursor, delimiter byte) (*token, cu
 					loc:   ic.loc,
 					kind:  stringKind,
 				}, cur, true
-			} else {
-				value = append(value, delimiter)
-				cur.pointer++
-				cur.loc.col++
 			}
+			value = append(value, delimiter)
+			cur.pointer++
+			cur.loc.col++
 		}
 
 		value = append(value, c)
@@ -440,7 +446,7 @@ func lexIdentifier(source string, ic cursor) (*token, cursor, bool) {
 	}
 
 	return &token{
-		// Unquoted dentifiers are case-insensitive
+		// Unquoted identifiers are case-insensitive
 		value: strings.ToLower(string(value)),
 		loc:   ic.loc,
 		kind:  identifierKind,
@@ -451,8 +457,20 @@ func lexString(source string, ic cursor) (*token, cursor, bool) {
 	return lexCharacterDelimited(source, ic, '\'')
 }
 
+type lexer func(string, cursor) (*token, cursor, bool)
+
+// lex splits an input string into a list of tokens. This process
+// can be divided into following tasks:
+//
+// 1. Instantiating a cursor with pointing to the start of the string
+//
+// 2. Execute all the lexers in series.
+//
+// 3. If any of the lexer generate a token then add the token to the
+// token slice, update the cursor and restart the process from the new
+// cursor location.
 func lex(source string) ([]*token, error) {
-	tokens := []*token{}
+	var tokens []*token
 	cur := cursor{}
 
 lex:

memory.go

@@ -9,6 +9,9 @@ import (
 	"github.com/petar/GoLLRB/llrb"
 )
 
+// memoryCell is the underlying storage for the in-memory backend
+// implementation. Each supported datatype can be mapped to and from
+// this byte array.
 type memoryCell []byte
 
 func (mc memoryCell) AsInt() *int32 {
@@ -79,9 +82,9 @@ func literalToMemoryCell(t *token) memoryCell {
 	if t.kind == boolKind {
 		if t.value == "true" {
 			return []byte{1}
-		} else {
-			return []byte{0}
 		}
+
+		return []byte{0}
 	}
 
 	return nil
@@ -531,9 +534,9 @@ type MemoryBackend struct {
 func (mb *MemoryBackend) Select(slct *SelectStatement) (*Results, error) {
 	t := createTable()
 
-	if slct.from != nil && slct.from.table != nil {
+	if slct.from != nil {
 		var ok bool
-		t, ok = mb.tables[slct.from.table.value]
+		t, ok = mb.tables[slct.from.value]
 		if !ok {
 			return nil, ErrTableDoesNotExist
 		}

memory_test.go

@@ -242,16 +242,16 @@ func TestTable_GetApplicableIndexes(t *testing.T) {
 }
 
 func TestLiteralToMemoryCell(t *testing.T) {
-	var i *int32 = nil
+	var i *int32
 	assert.Equal(t, i, literalToMemoryCell(&token{value: "null", kind: nullKind}).AsInt())
 	assert.Equal(t, i, literalToMemoryCell(&token{value: "not an int", kind: numericKind}).AsInt())
 	assert.Equal(t, int32(2), *literalToMemoryCell(&token{value: "2", kind: numericKind}).AsInt())
 
-	var s *string = nil
+	var s *string
 	assert.Equal(t, s, literalToMemoryCell(&token{value: "null", kind: nullKind}).AsText())
 	assert.Equal(t, "foo", *literalToMemoryCell(&token{value: "foo", kind: stringKind}).AsText())
 
-	var b *bool = nil
+	var b *bool
 	assert.Equal(t, b, literalToMemoryCell(&token{value: "null", kind: nullKind}).AsBool())
 	assert.Equal(t, true, *literalToMemoryCell(&token{value: "true", kind: boolKind}).AsBool())
 	assert.Equal(t, false, *literalToMemoryCell(&token{value: "false", kind: boolKind}).AsBool())

parser.go

@@ -23,6 +23,7 @@ type Parser struct {
 	HelpMessagesDisabled bool
 }
 
+// helpMessage prints errors found while parsing
 func (p Parser) helpMessage(tokens []*token, cursor uint, msg string) {
 	if p.HelpMessagesDisabled {
 		return
@@ -38,6 +39,7 @@ func (p Parser) helpMessage(tokens []*token, cursor uint, msg string) {
 	fmt.Printf("[%d,%d]: %s, near: %s\n", c.loc.line, c.loc.col, msg, c.value)
 }
 
+// parseTokenKind looks for a token of the given kind
 func (p Parser) parseTokenKind(tokens []*token, initialCursor uint, kind tokenKind) (*token, uint, bool) {
 	cursor := initialCursor
 
@@ -53,6 +55,8 @@ func (p Parser) parseTokenKind(tokens []*token, initialCursor uint, kind tokenKi
 	return nil, initialCursor, false
 }
 
+// parseToken looks for a token the same as passed in (ignoring token
+// location)
 func (p Parser) parseToken(tokens []*token, initialCursor uint, t token) (*token, uint, bool) {
 	cursor := initialCursor
 
@@ -134,7 +138,7 @@ outer:
 			tokenFromSymbol(plusSymbol),
 		}
 
-		var op *token = nil
+		var op *token
 		for _, bo := range binOps {
 			var t *token
 			t, cursor, ok = p.parseToken(tokens, cursor, bo)
@@ -175,11 +179,10 @@ outer:
 	return exp, cursor, true
 }
 
-// expression [AS ident] [, ...]
 func (p Parser) parseSelectItem(tokens []*token, initialCursor uint, delimiters []token) (*[]*selectItem, uint, bool) {
 	cursor := initialCursor
 
-	s := []*selectItem{}
+	var s []*selectItem
 outer:
 	for {
 		if cursor >= uint(len(tokens)) {
@@ -237,16 +240,6 @@ outer:
 	return &s, cursor, true
 }
 
-func (p Parser) parseFromItem(tokens []*token, initialCursor uint, _ []token) (*fromItem, uint, bool) {
-	ident, newCursor, ok := p.parseTokenKind(tokens, initialCursor, identifierKind)
-	if !ok {
-		return nil, initialCursor, false
-	}
-
-	return &fromItem{table: ident}, newCursor, true
-}
-
-// SELECT [ident [, ...]] [FROM ident] [WHERE condition [combinator ...]]
 func (p Parser) parseSelectStatement(tokens []*token, initialCursor uint, delimiter token) (*SelectStatement, uint, bool) {
 	var ok bool
 	cursor := initialCursor
@@ -267,11 +260,10 @@ func (p Parser) parseSelectStatement(tokens []*token, initialCursor uint, delimi
 	cursor = newCursor
 
 	whereToken := tokenFromKeyword(whereKeyword)
-	delimiters := []token{delimiter, whereToken}
 
 	_, cursor, ok = p.parseToken(tokens, cursor, fromToken)
 	if ok {
-		from, newCursor, ok := p.parseFromItem(tokens, cursor, delimiters)
+		from, newCursor, ok := p.parseTokenKind(tokens, cursor, identifierKind)
 		if !ok {
 			p.helpMessage(tokens, cursor, "Expected FROM item")
 			return nil, initialCursor, false
@@ -299,7 +291,7 @@ func (p Parser) parseSelectStatement(tokens []*token, initialCursor uint, delimi
 func (p Parser) parseExpressions(tokens []*token, initialCursor uint, delimiter token) (*[]*expression, uint, bool) {
 	cursor := initialCursor
 
-	exps := []*expression{}
+	var exps []*expression
 	for {
 		if cursor >= uint(len(tokens)) {
 			return nil, initialCursor, false
@@ -332,7 +324,7 @@ func (p Parser) parseExpressions(tokens []*token, initialCursor uint, delimiter
 	return &exps, cursor, true
 }
 
-func (p Parser) parseInsertStatement(tokens []*token, initialCursor uint, delimiter token) (*InsertStatement, uint, bool) {
+func (p Parser) parseInsertStatement(tokens []*token, initialCursor uint, _ token) (*InsertStatement, uint, bool) {
 	cursor := initialCursor
 	ok := false
 
@@ -388,7 +380,7 @@ func (p Parser) parseInsertStatement(tokens []*token, initialCursor uint, delimi
 func (p Parser) parseColumnDefinitions(tokens []*token, initialCursor uint, delimiter token) (*[]*columnDefinition, uint, bool) {
 	cursor := initialCursor
 
-	cds := []*columnDefinition{}
+	var cds []*columnDefinition
 	for {
 		if cursor >= uint(len(tokens)) {
 			return nil, initialCursor, false
@@ -438,7 +430,7 @@ func (p Parser) parseColumnDefinitions(tokens []*token, initialCursor uint, deli
 	return &cds, cursor, true
 }
 
-func (p Parser) parseCreateTableStatement(tokens []*token, initialCursor uint, delimiter token) (*CreateTableStatement, uint, bool) {
+func (p Parser) parseCreateTableStatement(tokens []*token, initialCursor uint, _ token) (*CreateTableStatement, uint, bool) {
 	cursor := initialCursor
 	ok := false
 
@@ -483,7 +475,7 @@ func (p Parser) parseCreateTableStatement(tokens []*token, initialCursor uint, d
 	}, cursor, true
 }
 
-func (p Parser) parseDropTableStatement(tokens []*token, initialCursor uint, delimiter token) (*DropTableStatement, uint, bool) {
+func (p Parser) parseDropTableStatement(tokens []*token, initialCursor uint, _ token) (*DropTableStatement, uint, bool) {
 	cursor := initialCursor
 	ok := false
 
@@ -509,7 +501,7 @@ func (p Parser) parseDropTableStatement(tokens []*token, initialCursor uint, del
 	}, cursor, true
 }
 
-func (p Parser) parseStatement(tokens []*token, initialCursor uint, delimiter token) (*Statement, uint, bool) {
+func (p Parser) parseStatement(tokens []*token, initialCursor uint, _ token) (*Statement, uint, bool) {
 	cursor := initialCursor
 
 	semicolonToken := tokenFromSymbol(semicolonSymbol)

parser_test.go

@@ -226,12 +226,10 @@ func TestParse(t *testing.T) {
 									},
 								},
 							},
-							from: &fromItem{
-								table: &token{
-									loc:   location{col: 33, line: 0},
-									kind:  identifierKind,
-									value: "users",
-								},
+							from: &token{
+								loc:   location{col: 33, line: 0},
+								kind:  identifierKind,
+								value: "users",
 							},
 						},
 					},