unicode: use new Scanner interface in table creation
Update norm and local/collate as well.
R=mpvl
CC=golang-dev
https://golang.org/cl/7395045
diff --git a/src/pkg/exp/locale/collate/maketables.go b/src/pkg/exp/locale/collate/maketables.go
index 19f176e..86feaf2 100644
--- a/src/pkg/exp/locale/collate/maketables.go
+++ b/src/pkg/exp/locale/collate/maketables.go
@@ -236,18 +236,10 @@
}
failOnError(err)
defer r.Close()
- input := bufio.NewReader(r)
+ scanner := bufio.NewScanner(r)
colelem := regexp.MustCompile(`\[([.*])([0-9A-F.]+)\]`)
- for i := 1; err == nil; i++ {
- l, prefix, e := input.ReadLine()
- err = e
- line := string(l)
- if prefix {
- log.Fatalf("%d: buffer overflow", i)
- }
- if err != nil && err != io.EOF {
- log.Fatalf("%d: %v", i, err)
- }
+ for i := 1; scanner.Scan(); i++ {
+ line := scanner.Text()
if len(line) == 0 || line[0] == '#' {
continue
}
@@ -300,6 +292,9 @@
failOnError(builder.Add(lhs, rhs, vars))
}
}
+ if scanner.Err() != nil {
+ log.Fatal(scanner.Err())
+ }
}
func convHex(line int, s string) int {
diff --git a/src/pkg/exp/locale/collate/regtest.go b/src/pkg/exp/locale/collate/regtest.go
index fd4fc6e..55c81f5 100644
--- a/src/pkg/exp/locale/collate/regtest.go
+++ b/src/pkg/exp/locale/collate/regtest.go
@@ -166,16 +166,10 @@
ff, err := f.Open()
Error(err)
defer ff.Close()
- input := bufio.NewReader(ff)
+ scanner := bufio.NewScanner(ff)
test := Test{name: path.Base(f.Name)}
- for {
- line, err := input.ReadString('\n')
- if err != nil {
- if err == io.EOF {
- break
- }
- log.Fatal(err)
- }
+ for scanner.Scan() {
+ line := scanner.Text()
if len(line) <= 1 || line[0] == '#' {
if m := versionRe.FindStringSubmatch(line); m != nil {
if m[1] != unicode.Version {
@@ -206,6 +200,9 @@
test.comment = append(test.comment, m[2])
}
}
+ if scanner.Err() != nil {
+ log.Fatal(scanner.Err())
+ }
tests = append(tests, test)
}
return tests
diff --git a/src/pkg/exp/norm/maketables.go b/src/pkg/exp/norm/maketables.go
index 03e1e2e..50c0c31 100644
--- a/src/pkg/exp/norm/maketables.go
+++ b/src/pkg/exp/norm/maketables.go
@@ -307,16 +307,12 @@
func loadUnicodeData() {
f := openReader("UnicodeData.txt")
defer f.Close()
- input := bufio.NewReader(f)
- for {
- line, err := input.ReadString('\n')
- if err != nil {
- if err == io.EOF {
- break
- }
- logger.Fatal(err)
- }
- parseCharacter(line[0 : len(line)-1])
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ parseCharacter(scanner.Text())
+ }
+ if scanner.Err() != nil {
+ logger.Fatal(scanner.Err())
}
}
@@ -347,16 +343,9 @@
func loadCompositionExclusions() {
f := openReader("CompositionExclusions.txt")
defer f.Close()
- input := bufio.NewReader(f)
- for {
- line, err := input.ReadString('\n')
- if err != nil {
- if err == io.EOF {
- break
- }
- logger.Fatal(err)
- }
- point := parseExclusion(line[0 : len(line)-1])
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ point := parseExclusion(scanner.Text())
if point == 0 {
continue
}
@@ -366,6 +355,9 @@
}
c.excludeInComp = true
}
+ if scanner.Err() != nil {
+ log.Fatal(scanner.Err())
+ }
}
// hasCompatDecomp returns true if any of the recursive
@@ -853,15 +845,9 @@
}
f := openReader("DerivedNormalizationProps.txt")
defer f.Close()
- input := bufio.NewReader(f)
- for {
- line, err := input.ReadString('\n')
- if err != nil {
- if err == io.EOF {
- break
- }
- logger.Fatal(err)
- }
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ line := scanner.Text()
qc := qcRe.FindStringSubmatch(line)
if qc == nil {
continue
@@ -920,6 +906,9 @@
}
}
}
+ if scanner.Err() != nil {
+ logger.Fatal(scanner.Err())
+ }
// Any unspecified value must be QCYes. Verify this.
for i, c := range chars {
for j, fd := range c.forms {
diff --git a/src/pkg/exp/norm/normregtest.go b/src/pkg/exp/norm/normregtest.go
index d7ef9ba..b77b5b5 100644
--- a/src/pkg/exp/norm/normregtest.go
+++ b/src/pkg/exp/norm/normregtest.go
@@ -12,7 +12,6 @@
"exp/norm"
"flag"
"fmt"
- "io"
"log"
"net/http"
"os"
@@ -118,8 +117,8 @@
return fmt.Sprintf("%s:%d", part[t.partnr].name, t.number)
}
-var partRe = regexp.MustCompile(`@Part(\d) # (.*)\n$`)
-var testRe = regexp.MustCompile(`^` + strings.Repeat(`([\dA-F ]+);`, 5) + ` # (.*)\n?$`)
+var partRe = regexp.MustCompile(`@Part(\d) # (.*)$`)
+var testRe = regexp.MustCompile(`^` + strings.Repeat(`([\dA-F ]+);`, 5) + ` # (.*)$`)
var counter int
@@ -141,15 +140,9 @@
}
f := resp.Body
defer f.Close()
- input := bufio.NewReader(f)
- for {
- line, err := input.ReadString('\n')
- if err != nil {
- if err == io.EOF {
- break
- }
- logger.Fatal(err)
- }
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ line := scanner.Text()
if len(line) == 0 || line[0] == '#' {
continue
}
@@ -190,6 +183,9 @@
part := &part[len(part)-1]
part.tests = append(part.tests, test)
}
+ if scanner.Err() != nil {
+ logger.Fatal(scanner.Err())
+ }
}
var fstr = []string{"NFC", "NFD", "NFKC", "NFKD"}
diff --git a/src/pkg/unicode/maketables.go b/src/pkg/unicode/maketables.go
index 915101d..53d8b96 100644
--- a/src/pkg/unicode/maketables.go
+++ b/src/pkg/unicode/maketables.go
@@ -13,7 +13,6 @@
"bufio"
"flag"
"fmt"
- "io"
"log"
"net/http"
"os"
@@ -321,16 +320,11 @@
flag.Set("data", *url+"UnicodeData.txt")
}
input := open(*dataURL)
+ defer input.close()
+ scanner := bufio.NewScanner(input)
var first rune = 0
- for {
- line, err := input.ReadString('\n')
- if err != nil {
- if err == io.EOF {
- break
- }
- logger.Fatal(err)
- }
- switch parseCategory(line[0 : len(line)-1]) {
+ for scanner.Scan() {
+ switch parseCategory(scanner.Text()) {
case SNormal:
if first != 0 {
logger.Fatalf("bad state normal at %U", lastChar)
@@ -351,7 +345,9 @@
first = 0
}
}
- input.close()
+ if scanner.Err() != nil {
+ logger.Fatal(scanner.Err())
+ }
}
func loadCasefold() {
@@ -359,15 +355,11 @@
flag.Set("casefolding", *url+"CaseFolding.txt")
}
input := open(*casefoldingURL)
- for {
- line, err := input.ReadString('\n')
- if err != nil {
- if err == io.EOF {
- break
- }
- logger.Fatal(err)
- }
- if line[0] == '#' || len(strings.TrimSpace(line)) == 0 {
+ defer input.close()
+ scanner := bufio.NewScanner(input)
+ for scanner.Scan() {
+ line := scanner.Text()
+ if len(line) == 0 || line[0] == '#' || len(strings.TrimSpace(line)) == 0 {
continue
}
field := strings.Split(line, "; ")
@@ -389,7 +381,9 @@
}
chars[p1].foldCase = rune(p2)
}
- input.close()
+ if scanner.Err() != nil {
+ logger.Fatal(scanner.Err())
+ }
}
const progHeader = `// Generated by running
@@ -710,15 +704,12 @@
return
}
input := open(*url + file)
- for {
- line, err := input.ReadString('\n')
- if err != nil {
- if err == io.EOF {
- break
- }
- logger.Fatal(err)
- }
- parseScript(line[0:len(line)-1], table)
+ scanner := bufio.NewScanner(input)
+ for scanner.Scan() {
+ parseScript(scanner.Text(), table)
+ }
+ if scanner.Err() != nil {
+ logger.Fatal(scanner.Err())
}
input.close()
