constants and formats

youngphp · Mar 9, 2013 · e523a50 · e523a50
1 parent 42c5677
commit e523a50
Show file tree

Hide file tree

Showing 12 changed files with 162 additions and 119 deletions.
diff --git a/text/README.md b/text/README.md
@@ -1,6 +1,6 @@
 # 包名
 
-函数列表
+包列表
 
-- xxx1
-- xxx2
+- [text/scanner](scanner)
+- xxx2
diff --git a/text/scanner/Constants.md b/text/scanner/Constants.md
@@ -0,0 +1,33 @@
+	const (
+    	ScanIdents     = 1 << -Ident
+    	ScanInts       = 1 << -Int
+    	ScanFloats     = 1 << -Float // includes Ints
+    	ScanChars      = 1 << -Char
+    	ScanStrings    = 1 << -String
+    	ScanRawStrings = 1 << -RawString
+    	ScanComments   = 1 << -Comment
+    	SkipComments   = 1 << -skipComment // if set with ScanComments, comments become white space
+    	GoTokens       = ScanIdents | ScanFloats | ScanChars | ScanStrings | ScanRawStrings | ScanComments | SkipComments
+	)
+
+用于控制词法单元识别的预定义模式位。比如，通过以下模式位设置，可以配置一个只识别（Go）标识符、整型、并且忽略注释的词法解析器：
+
+	ScanIdents | ScanInts | SkipComments
+
+	const (
+    	EOF = -(iota + 1)
+    	Ident
+    	Int
+    	Float
+    	Char
+    	String
+    	RawString
+    	Comment
+	)
+
+通过Scan方法可以获取的词法单元，此外还可以获取Unicode字符。
+
+	const GoWhitespace = 1<<'\t' | 1<<'\n' | 1<<'\r' | 1<<' '
+
+GoWhitespace，Scanner的空白字符的默认值。
+
diff --git a/text/scanner/Init.md b/text/scanner/Init.md
@@ -14,28 +14,28 @@
 
 代码实例：
 
-package main
+	package main
 
-import(
-	"fmt"
-	"strings"
-	"text/scanner"
-)
+	import(
+		"fmt"
+		"strings"
+		"text/scanner"
+	)
 
-func main(){
+	func main(){
 
-	src := strings.NewReader("int hello = 3; hello+23; print hello;")
+		src := strings.NewReader("int hello = 3; hello+23; print hello;")
 
-	fmt.Println(src)
+		fmt.Println(src)
 
-	var s scanner.Scanner
-	s.Init(src)
+		var s scanner.Scanner
+		s.Init(src)
 
-	tok := s.Scan()
-	fmt.Println(s.TokenText())
-	for tok != scanner.EOF{
-		tok = s.Scan()
+		tok := s.Scan()
 		fmt.Println(s.TokenText())
+		for tok != scanner.EOF{
+			tok = s.Scan()
+		fmt.Println(s.TokenText())
+		}
 	}
-}
 
diff --git a/text/scanner/IsValid.md b/text/scanner/IsValid.md
@@ -14,25 +14,25 @@
 
 代码实例：
 
-package main
+	package main
 
-import(
-	"fmt"
-	"strings"
-	"text/scanner"
-)
+	import(
+		"fmt"
+		"strings"
+		"text/scanner"
+	)
 
-func main(){
+	func main(){
 
-	src := strings.NewReader("int num = 1;")
-	var s scanner.Scanner
+		src := strings.NewReader("int num = 1;")
+		var s scanner.Scanner
 
-	s.Init(src)
+		s.Init(src)
 
-	s.Next()
+		s.Next()
 
-	pos := s.Pos()
-	fmt.Println(pos.IsValid())
+		pos := s.Pos()
+		fmt.Println(pos.IsValid())
 
-}
+	}
 
diff --git a/text/scanner/Next.md b/text/scanner/Next.md
@@ -14,24 +14,24 @@
 
 代码实例：
 
-package main
+	package main
 
-import (
-	"fmt"
-	"strings"
-	"text/scanner"
-)
+	import (
+		"fmt"
+		"strings"
+		"text/scanner"
+	)
 
-func main() {
+	func main() {
 
-	src := strings.NewReader("int num = 1;")
+		src := strings.NewReader("int num = 1;")
 
-	var s scanner.Scanner
+		var s scanner.Scanner
 
-	s.Init(src)
+		s.Init(src)
 
-	//this will print "i" to stdout
-	fmt.Println(scanner.TokenString(s.Next()))
+		//this will print "i" to stdout
+		fmt.Println(scanner.TokenString(s.Next()))
 
-}
+	}
 
diff --git a/text/scanner/Peek.md b/text/scanner/Peek.md
@@ -14,22 +14,22 @@
 
 代码实例：
 
-package main
+	package main
 
-import(
-	"fmt"
-	"strings"
-	"text/scanner"
-)
+	import(
+		"fmt"
+		"strings"
+		"text/scanner"
+	)
 
-func main(){
+	func main(){
 
-	src := strings.NewReader("int num = 1;")
+		src := strings.NewReader("int num = 1;")
 
-	var s scanner.Scanner
-	s.Init(src)
+		var s scanner.Scanner
+		s.Init(src)
 
-	//this will print "i" to stdout
-	fmt.Println(scanner.TokenString(s.Peek()))
-}
+		//this will print "i" to stdout
+		fmt.Println(scanner.TokenString(s.Peek()))
+	}
 
diff --git a/text/scanner/Position.md b/text/scanner/Position.md
@@ -0,0 +1,9 @@
+	type Position struct {
+    	Filename string // 文件名，可能为空
+    	Offset   int    // 位移，从0计
+    	Line     int    // 行数，从1计
+    	Column   int    // 列数，从1计（按字符计算）
+	}
+
+Position用来表示源中的位置。行数Line > 0的情况下，Position合法。
+
diff --git a/text/scanner/README.md b/text/scanner/README.md
@@ -1,11 +1,12 @@
 # text/scanner包函数列表
 
-Constants
+- [Constants](Constants.md)
 - [func TokenString(tok rune) string](TokenString.md)
-type Position
+- [type Position](Position.md)
     - [func (pos *Position) IsValid() bool](IsValid.md)
     - [func (pos Position) String() string](String.md)
-type Scanner
+- type Scanner
+	Scanner用于从io.Reader中读取Unicode字符或词法单元
     - [func (s *Scanner) Init(src io.Reader) *Scanner](Init.md)
     - [func (s *Scanner) Next() rune](Next.md)
     - [func (s *Scanner) Peek() rune](Peek.md)

diff --git a/text/scanner/Scan.md b/text/scanner/Scan.md
@@ -14,23 +14,23 @@ Scan方法读取源中下一个token或字符，并返回。如果读到源的
 
 代码实例：
 
-package main
+	package main
 
-import(
-	"fmt"
-	"strings"
-	"text/scanner"
-)
+	import(
+		"fmt"
+		"strings"
+		"text/scanner"
+	)
 
-func main(){
+	func main(){
 
-	src := strings.NewReader("int num = 1;")
-	var s scanner.Scanner
+		src := strings.NewReader("int num = 1;")
+		var s scanner.Scanner
 
-	s.Init(src)
+		s.Init(src)
 
-	s.Scan()
-	//this will print the next token "int "to stdout
-	fmt.Println(s.TokenText())
-}
+		s.Scan()
+		//this will print the next token "int "to stdout
+		fmt.Println(s.TokenText())
+	}
 
diff --git a/text/scanner/String.md b/text/scanner/String.md
@@ -14,27 +14,27 @@
 
 代码实例：
 
-package main
+	package main
 
-import(
-	"fmt"
-	"strings"
-	"text/scanner"
-)
+	import(
+		"fmt"
+		"strings"
+		"text/scanner"
+	)
 
-func main(){
+	func main(){
 
-	src := strings.NewReader("int num = 1;")
-	var s scanner.Scanner
+		src := strings.NewReader("int num = 1;")
+		var s scanner.Scanner
 
-	s.Init(src)
+		s.Init(src)
 
-	s.Scan()
-	//this will print "1:4" to stdout
-	fmt.Println(s.Pos().String())
+		s.Scan()
+		//this will print "1:4" to stdout
+		fmt.Println(s.Pos().String())
 
-	s.Next()
-	//this will print "1:5" to stdout
-	fmt.Println(s.Pos().String())
-}
+		s.Next()
+		//this will print "1:5" to stdout
+		fmt.Println(s.Pos().String())
+	}
 
diff --git a/text/scanner/TokenString.md b/text/scanner/TokenString.md
@@ -14,18 +14,18 @@
 
 代码实例：
 
-package main
+	package main
 
-import(
-	"fmt"
-	"text/scanner"
-)
-func main(){
+	import(
+		"fmt"
+		"text/scanner"
+	)
+	func main(){
 
-	toks := []rune("abcdef")
+		toks := []rune("abcdef")
 
-	for _, tok := range toks{
-		fmt.Print(scanner.TokenString(tok))
+		for _, tok := range toks{
+			fmt.Print(scanner.TokenString(tok))
+		}
 	}
-}
 
diff --git a/text/scanner/TokenText.md b/text/scanner/TokenText.md
@@ -14,29 +14,29 @@
 
 代码实例：
 
-package main
+	package main
 
-import(
-	"fmt"
-	"strings"
-	"text/scanner"
-)
+	import(
+		"fmt"
+		"strings"
+		"text/scanner"
+	)
 
-func main(){
+	func main(){
 
-	src := strings.NewReader("int num = 1;")
-	var s scanner.Scanner
+		src := strings.NewReader("int num = 1;")
+		var s scanner.Scanner
 
-	s.Init(src)
+		s.Init(src)
 
-	//this will print nothing
-	fmt.Print(s.TokenText())
+		//this will print nothing
+		fmt.Print(s.TokenText())
 
-	s.Scan()
+		s.Scan()
 
-	//this will print "int" to stdout
-	fmt.Print(s.TokenText())
+		//this will print "int" to stdout
+		fmt.Print(s.TokenText())
 
-}
+	}