text/scanner: add examples
Added examples for use of Mode, Whitespace, and IsIdentRune properties.
Fixes #23768
Change-Id: I2528e14fde63a4476f3c25510bf0c5b73f38ba5d
Reviewed-on: https://go-review.googlesource.com/93199
Run-TryBot: Robert Griesemer <gri@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>
diff --git a/src/text/scanner/example_test.go b/src/text/scanner/example_test.go
index 97e22a9..5e8c3fb 100644
--- a/src/text/scanner/example_test.go
+++ b/src/text/scanner/example_test.go
@@ -1,4 +1,4 @@
-// Copyright 2015 The Go Authors. All rights reserved.
+// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
@@ -8,6 +8,7 @@
"fmt"
"strings"
"text/scanner"
+ "unicode"
)
func Example() {
@@ -16,6 +17,7 @@
if a > 10 {
someParsable = text
}`
+
var s scanner.Scanner
s.Init(strings.NewReader(src))
s.Filename = "example"
@@ -34,3 +36,105 @@
// example:4:17: text
// example:5:1: }
}
+
+func Example_isIdentRune() {
+ const src = "%var1 var2%"
+
+ var s scanner.Scanner
+ s.Init(strings.NewReader(src))
+ s.Filename = "default"
+
+ for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
+ fmt.Printf("%s: %s\n", s.Position, s.TokenText())
+ }
+
+ fmt.Println()
+ s.Init(strings.NewReader(src))
+ s.Filename = "percent"
+
+ // treat leading '%' as part of an identifier
+ s.IsIdentRune = func(ch rune, i int) bool {
+ return ch == '%' && i == 0 || unicode.IsLetter(ch) || unicode.IsDigit(ch) && i > 0
+ }
+
+ for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
+ fmt.Printf("%s: %s\n", s.Position, s.TokenText())
+ }
+
+ // Output:
+ // default:1:1: %
+ // default:1:2: var1
+ // default:1:7: var2
+ // default:1:11: %
+ //
+ // percent:1:1: %var1
+ // percent:1:7: var2
+ // percent:1:11: %
+}
+
+func Example_mode() {
+ const src = `
+ // Comment begins at column 5.
+
+This line should not be included in the output.
+
+/*
+This multiline comment
+should be extracted in
+its entirety.
+*/
+`
+
+ var s scanner.Scanner
+ s.Init(strings.NewReader(src))
+ s.Filename = "comments"
+ s.Mode ^= scanner.SkipComments // don't skip comments
+
+ for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
+ txt := s.TokenText()
+ if strings.HasPrefix(txt, "//") || strings.HasPrefix(txt, "/*") {
+ fmt.Printf("%s: %s\n", s.Position, txt)
+ }
+ }
+
+ // Output:
+ // comments:2:5: // Comment begins at column 5.
+ // comments:6:1: /*
+ // This multiline comment
+ // should be extracted in
+ // its entirety.
+ // */
+}
+
+func Example_whitespace() {
+ // tab-separated values
+ const src = `aa ab ac ad
+ba bb bc bd
+ca cb cc cd
+da db dc dd`
+
+ var (
+ col, row int
+ s scanner.Scanner
+ tsv [4][4]string // large enough for example above
+ )
+ s.Init(strings.NewReader(src))
+ s.Whitespace ^= 1<<'\t' | 1<<'\n' // don't skip tabs and new lines
+
+ for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
+ switch tok {
+ case '\n':
+ row++
+ col = 0
+ case '\t':
+ col++
+ default:
+ tsv[row][col] = s.TokenText()
+ }
+ }
+
+ fmt.Print(tsv)
+
+ // Output:
+ // [[aa ab ac ad] [ba bb bc bd] [ca cb cc cd] [da db dc dd]]
+}
