text/scanner: accept new Go2 number literals
This CL introduces text/scanner support for the new binary and octal integer
literals, hexadecimal floats, and digit separators for all number literals.
The new code is closely mirroring the respective code for number literals in
cmd/compile/internal/syntax/scanner.go.
Uniformly use the term "invalid" rather than "illegal" in error messages
to match the respective error messages in the other scanners directly.
R=Go1.13
Updates #12711.
Updates #19308.
Updates #28493.
Updates #29008.
Change-Id: I2f291de13ba5afc0e530cd8326e6bf4c3858ebac
Reviewed-on: https://go-review.googlesource.com/c/161199
Run-TryBot: Robert Griesemer <gri@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Ian Lance Taylor <iant@golang.org>
diff --git a/src/text/scanner/scanner.go b/src/text/scanner/scanner.go
index 62b3231..38c27f6 100644
--- a/src/text/scanner/scanner.go
+++ b/src/text/scanner/scanner.go
@@ -266,7 +266,7 @@
s.srcPos += width
s.lastCharLen = width
s.column++
- s.error("illegal UTF-8 encoding")
+ s.error("invalid UTF-8 encoding")
return ch
}
}
@@ -281,7 +281,7 @@
switch ch {
case 0:
// for compatibility with other tools
- s.error("illegal character NUL")
+ s.error("invalid character NUL")
case '\n':
s.line++
s.lastLineLen = s.column
@@ -335,6 +335,10 @@
fmt.Fprintf(os.Stderr, "%s: %s\n", pos, msg)
}
+func (s *Scanner) errorf(format string, args ...interface{}) {
+ s.error(fmt.Sprintf(format, args...))
+}
+
func (s *Scanner) isIdentRune(ch rune, i int) bool {
if s.IsIdentRune != nil {
return s.IsIdentRune(ch, i)
@@ -351,104 +355,198 @@
return ch
}
+func lower(ch rune) rune { return ('a' - 'A') | ch } // returns lower-case ch iff ch is ASCII letter
+func isDecimal(ch rune) bool { return '0' <= ch && ch <= '9' }
+func isHex(ch rune) bool { return '0' <= ch && ch <= '9' || 'a' <= lower(ch) && lower(ch) <= 'f' }
+
+// digits accepts the sequence { digit | '_' } starting with ch0.
+// If base <= 10, digits accepts any decimal digit but records
+// the first invalid digit >= base in *invalid if *invalid == 0.
+// digits returns the first rune that is not part of the sequence
+// anymore, and a bitset describing whether the sequence contained
+// digits (bit 0 is set), or separators '_' (bit 1 is set).
+func (s *Scanner) digits(ch0 rune, base int, invalid *rune) (ch rune, digsep int) {
+ ch = ch0
+ if base <= 10 {
+ max := rune('0' + base)
+ for isDecimal(ch) || ch == '_' {
+ ds := 1
+ if ch == '_' {
+ ds = 2
+ } else if ch >= max && *invalid == 0 {
+ *invalid = ch
+ }
+ digsep |= ds
+ ch = s.next()
+ }
+ } else {
+ for isHex(ch) || ch == '_' {
+ ds := 1
+ if ch == '_' {
+ ds = 2
+ }
+ digsep |= ds
+ ch = s.next()
+ }
+ }
+ return
+}
+
+func (s *Scanner) scanNumber(ch rune, integerPart bool) (rune, rune) {
+ base := 10 // number base
+ prefix := rune(0) // one of 0 (decimal), '0' (0-octal), 'x', 'o', or 'b'
+ digsep := 0 // bit 0: digit present, bit 1: '_' present
+ invalid := rune(0) // invalid digit in literal, or 0
+
+ // integer part
+ var tok rune
+ var ds int
+ if integerPart {
+ tok = Int
+ if ch == '0' {
+ ch = s.next()
+ switch lower(ch) {
+ case 'x':
+ ch = s.next()
+ base, prefix = 16, 'x'
+ case 'o':
+ ch = s.next()
+ base, prefix = 8, 'o'
+ case 'b':
+ ch = s.next()
+ base, prefix = 2, 'b'
+ default:
+ base, prefix = 8, '0'
+ digsep = 1 // leading 0
+ }
+ }
+ ch, ds = s.digits(ch, base, &invalid)
+ digsep |= ds
+ }
+
+ // fractional part
+ if !integerPart || ch == '.' {
+ tok = Float
+ if prefix == 'o' || prefix == 'b' {
+ s.error("invalid radix point in " + litname(prefix))
+ }
+ if ch == '.' {
+ ch = s.next()
+ }
+ ch, ds = s.digits(ch, base, &invalid)
+ digsep |= ds
+ }
+
+ if digsep&1 == 0 {
+ s.error(litname(prefix) + " has no digits")
+ }
+
+ // exponent
+ if e := lower(ch); e == 'e' || e == 'p' {
+ switch {
+ case e == 'e' && prefix != 0 && prefix != '0':
+ s.errorf("%q exponent requires decimal mantissa", ch)
+ case e == 'p' && prefix != 'x':
+ s.errorf("%q exponent requires hexadecimal mantissa", ch)
+ }
+ ch = s.next()
+ tok = Float
+ if ch == '+' || ch == '-' {
+ ch = s.next()
+ }
+ ch, ds = s.digits(ch, 10, nil)
+ digsep |= ds
+ if ds&1 == 0 {
+ s.error("exponent has no digits")
+ }
+ } else if prefix == 'x' && tok == Float {
+ s.error("hexadecimal mantissa requires a 'p' exponent")
+ }
+
+ if tok == Int && invalid != 0 {
+ s.errorf("invalid digit %q in %s", invalid, litname(prefix))
+ }
+
+ if digsep&2 != 0 {
+ s.tokEnd = s.srcPos - s.lastCharLen // make sure token text is terminated
+ if i := invalidSep(s.TokenText()); i >= 0 {
+ s.error("'_' must separate successive digits")
+ }
+ }
+
+ return tok, ch
+}
+
+func litname(prefix rune) string {
+ switch prefix {
+ default:
+ return "decimal literal"
+ case 'x':
+ return "hexadecimal literal"
+ case 'o', '0':
+ return "octal literal"
+ case 'b':
+ return "binary literal"
+ }
+}
+
+// invalidSep returns the index of the first invalid separator in x, or -1.
+func invalidSep(x string) int {
+ x1 := ' ' // prefix char, we only care if it's 'x'
+ d := '.' // digit, one of '_', '0' (a digit), or '.' (anything else)
+ i := 0
+
+ // a prefix counts as a digit
+ if len(x) >= 2 && x[0] == '0' {
+ x1 = lower(rune(x[1]))
+ if x1 == 'x' || x1 == 'o' || x1 == 'b' {
+ d = '0'
+ i = 2
+ }
+ }
+
+ // mantissa and exponent
+ for ; i < len(x); i++ {
+ p := d // previous digit
+ d = rune(x[i])
+ switch {
+ case d == '_':
+ if p != '0' {
+ return i
+ }
+ case isDecimal(d) || x1 == 'x' && isHex(d):
+ d = '0'
+ default:
+ if p == '_' {
+ return i - 1
+ }
+ d = '.'
+ }
+ }
+ if d == '_' {
+ return len(x) - 1
+ }
+
+ return -1
+}
+
func digitVal(ch rune) int {
switch {
case '0' <= ch && ch <= '9':
return int(ch - '0')
- case 'a' <= ch && ch <= 'f':
- return int(ch - 'a' + 10)
- case 'A' <= ch && ch <= 'F':
- return int(ch - 'A' + 10)
+ case 'a' <= lower(ch) && lower(ch) <= 'f':
+ return int(lower(ch) - 'a' + 10)
}
return 16 // larger than any legal digit val
}
-func isDecimal(ch rune) bool { return '0' <= ch && ch <= '9' }
-
-func (s *Scanner) scanMantissa(ch rune) rune {
- for isDecimal(ch) {
- ch = s.next()
- }
- return ch
-}
-
-func (s *Scanner) scanFraction(ch rune) rune {
- if ch == '.' {
- ch = s.scanMantissa(s.next())
- }
- return ch
-}
-
-func (s *Scanner) scanExponent(ch rune) rune {
- if ch == 'e' || ch == 'E' {
- ch = s.next()
- if ch == '-' || ch == '+' {
- ch = s.next()
- }
- if !isDecimal(ch) {
- s.error("illegal exponent")
- }
- ch = s.scanMantissa(ch)
- }
- return ch
-}
-
-func (s *Scanner) scanNumber(ch rune) (rune, rune) {
- // isDecimal(ch)
- if ch == '0' {
- // int or float
- ch = s.next()
- if ch == 'x' || ch == 'X' {
- // hexadecimal int
- ch = s.next()
- hasMantissa := false
- for digitVal(ch) < 16 {
- ch = s.next()
- hasMantissa = true
- }
- if !hasMantissa {
- s.error("illegal hexadecimal number")
- }
- } else {
- // octal int or float
- has8or9 := false
- for isDecimal(ch) {
- if ch > '7' {
- has8or9 = true
- }
- ch = s.next()
- }
- if s.Mode&ScanFloats != 0 && (ch == '.' || ch == 'e' || ch == 'E') {
- // float
- ch = s.scanFraction(ch)
- ch = s.scanExponent(ch)
- return Float, ch
- }
- // octal int
- if has8or9 {
- s.error("illegal octal number")
- }
- }
- return Int, ch
- }
- // decimal int or float
- ch = s.scanMantissa(ch)
- if s.Mode&ScanFloats != 0 && (ch == '.' || ch == 'e' || ch == 'E') {
- // float
- ch = s.scanFraction(ch)
- ch = s.scanExponent(ch)
- return Float, ch
- }
- return Int, ch
-}
-
func (s *Scanner) scanDigits(ch rune, base, n int) rune {
for n > 0 && digitVal(ch) < base {
ch = s.next()
n--
}
if n > 0 {
- s.error("illegal char escape")
+ s.error("invalid char escape")
}
return ch
}
@@ -468,7 +566,7 @@
case 'U':
ch = s.scanDigits(s.next(), 16, 8)
default:
- s.error("illegal char escape")
+ s.error("invalid char escape")
}
return ch
}
@@ -503,7 +601,7 @@
func (s *Scanner) scanChar() {
if s.scanString('\'') != 1 {
- s.error("illegal char literal")
+ s.error("invalid char literal")
}
}
@@ -584,7 +682,7 @@
}
case isDecimal(ch):
if s.Mode&(ScanInts|ScanFloats) != 0 {
- tok, ch = s.scanNumber(ch)
+ tok, ch = s.scanNumber(ch, true)
} else {
ch = s.next()
}
@@ -607,9 +705,7 @@
case '.':
ch = s.next()
if isDecimal(ch) && s.Mode&ScanFloats != 0 {
- tok = Float
- ch = s.scanMantissa(ch)
- ch = s.scanExponent(ch)
+ tok, ch = s.scanNumber(ch, false)
}
case '/':
ch = s.next()
diff --git a/src/text/scanner/scanner_test.go b/src/text/scanner/scanner_test.go
index e7539a0..58db8e1 100644
--- a/src/text/scanner/scanner_test.go
+++ b/src/text/scanner/scanner_test.go
@@ -290,11 +290,11 @@
testScan(t, GoTokens&^SkipComments)
}
-func TestIllegalExponent(t *testing.T) {
+func TestInvalidExponent(t *testing.T) {
const src = "1.5e 1.5E 1e+ 1e- 1.5z"
s := new(Scanner).Init(strings.NewReader(src))
s.Error = func(s *Scanner, msg string) {
- const want = "illegal exponent"
+ const want = "exponent has no digits"
if msg != want {
t.Errorf("%s: got error %q; want %q", s.TokenText(), msg, want)
}
@@ -378,7 +378,7 @@
testScanSelectedMode(t, 0, 0)
testScanSelectedMode(t, ScanIdents, Ident)
// Don't test ScanInts and ScanNumbers since some parts of
- // the floats in the source look like (illegal) octal ints
+ // the floats in the source look like (invalid) octal ints
// and ScanNumbers may return either Int or Float.
testScanSelectedMode(t, ScanChars, Char)
testScanSelectedMode(t, ScanStrings, String)
@@ -480,34 +480,34 @@
}
func TestError(t *testing.T) {
- testError(t, "\x00", "<input>:1:1", "illegal character NUL", 0)
- testError(t, "\x80", "<input>:1:1", "illegal UTF-8 encoding", utf8.RuneError)
- testError(t, "\xff", "<input>:1:1", "illegal UTF-8 encoding", utf8.RuneError)
+ testError(t, "\x00", "<input>:1:1", "invalid character NUL", 0)
+ testError(t, "\x80", "<input>:1:1", "invalid UTF-8 encoding", utf8.RuneError)
+ testError(t, "\xff", "<input>:1:1", "invalid UTF-8 encoding", utf8.RuneError)
- testError(t, "a\x00", "<input>:1:2", "illegal character NUL", Ident)
- testError(t, "ab\x80", "<input>:1:3", "illegal UTF-8 encoding", Ident)
- testError(t, "abc\xff", "<input>:1:4", "illegal UTF-8 encoding", Ident)
+ testError(t, "a\x00", "<input>:1:2", "invalid character NUL", Ident)
+ testError(t, "ab\x80", "<input>:1:3", "invalid UTF-8 encoding", Ident)
+ testError(t, "abc\xff", "<input>:1:4", "invalid UTF-8 encoding", Ident)
- testError(t, `"a`+"\x00", "<input>:1:3", "illegal character NUL", String)
- testError(t, `"ab`+"\x80", "<input>:1:4", "illegal UTF-8 encoding", String)
- testError(t, `"abc`+"\xff", "<input>:1:5", "illegal UTF-8 encoding", String)
+ testError(t, `"a`+"\x00", "<input>:1:3", "invalid character NUL", String)
+ testError(t, `"ab`+"\x80", "<input>:1:4", "invalid UTF-8 encoding", String)
+ testError(t, `"abc`+"\xff", "<input>:1:5", "invalid UTF-8 encoding", String)
- testError(t, "`a"+"\x00", "<input>:1:3", "illegal character NUL", RawString)
- testError(t, "`ab"+"\x80", "<input>:1:4", "illegal UTF-8 encoding", RawString)
- testError(t, "`abc"+"\xff", "<input>:1:5", "illegal UTF-8 encoding", RawString)
+ testError(t, "`a"+"\x00", "<input>:1:3", "invalid character NUL", RawString)
+ testError(t, "`ab"+"\x80", "<input>:1:4", "invalid UTF-8 encoding", RawString)
+ testError(t, "`abc"+"\xff", "<input>:1:5", "invalid UTF-8 encoding", RawString)
- testError(t, `'\"'`, "<input>:1:3", "illegal char escape", Char)
- testError(t, `"\'"`, "<input>:1:3", "illegal char escape", String)
+ testError(t, `'\"'`, "<input>:1:3", "invalid char escape", Char)
+ testError(t, `"\'"`, "<input>:1:3", "invalid char escape", String)
- testError(t, `01238`, "<input>:1:6", "illegal octal number", Int)
- testError(t, `01238123`, "<input>:1:9", "illegal octal number", Int)
- testError(t, `0x`, "<input>:1:3", "illegal hexadecimal number", Int)
- testError(t, `0xg`, "<input>:1:3", "illegal hexadecimal number", Int)
- testError(t, `'aa'`, "<input>:1:4", "illegal char literal", Char)
- testError(t, `1.5e`, "<input>:1:5", "illegal exponent", Float)
- testError(t, `1.5E`, "<input>:1:5", "illegal exponent", Float)
- testError(t, `1.5e+`, "<input>:1:6", "illegal exponent", Float)
- testError(t, `1.5e-`, "<input>:1:6", "illegal exponent", Float)
+ testError(t, `01238`, "<input>:1:6", "invalid digit '8' in octal literal", Int)
+ testError(t, `01238123`, "<input>:1:9", "invalid digit '8' in octal literal", Int)
+ testError(t, `0x`, "<input>:1:3", "hexadecimal literal has no digits", Int)
+ testError(t, `0xg`, "<input>:1:3", "hexadecimal literal has no digits", Int)
+ testError(t, `'aa'`, "<input>:1:4", "invalid char literal", Char)
+ testError(t, `1.5e`, "<input>:1:5", "exponent has no digits", Float)
+ testError(t, `1.5E`, "<input>:1:5", "exponent has no digits", Float)
+ testError(t, `1.5e+`, "<input>:1:6", "exponent has no digits", Float)
+ testError(t, `1.5e-`, "<input>:1:6", "exponent has no digits", Float)
testError(t, `'`, "<input>:1:2", "literal not terminated", Char)
testError(t, `'`+"\n", "<input>:1:2", "literal not terminated", Char)
@@ -711,3 +711,169 @@
for r := s.Scan(); r != EOF; r = s.Scan() {
}
}
+
+func TestNumbers(t *testing.T) {
+ for _, test := range []struct {
+ tok rune
+ src, tokens, err string
+ }{
+ // binaries
+ {Int, "0b0", "0b0", ""},
+ {Int, "0b1010", "0b1010", ""},
+ {Int, "0B1110", "0B1110", ""},
+
+ {Int, "0b", "0b", "binary literal has no digits"},
+ {Int, "0b0190", "0b0190", "invalid digit '9' in binary literal"},
+ {Int, "0b01a0", "0b01 a0", ""}, // only accept 0-9
+
+ // binary floats (invalid)
+ {Float, "0b.", "0b.", "invalid radix point in binary literal"},
+ {Float, "0b.1", "0b.1", "invalid radix point in binary literal"},
+ {Float, "0b1.0", "0b1.0", "invalid radix point in binary literal"},
+ {Float, "0b1e10", "0b1e10", "'e' exponent requires decimal mantissa"},
+ {Float, "0b1P-1", "0b1P-1", "'P' exponent requires hexadecimal mantissa"},
+
+ // octals
+ {Int, "0o0", "0o0", ""},
+ {Int, "0o1234", "0o1234", ""},
+ {Int, "0O1234", "0O1234", ""},
+
+ {Int, "0o", "0o", "octal literal has no digits"},
+ {Int, "0o8123", "0o8123", "invalid digit '8' in octal literal"},
+ {Int, "0o1293", "0o1293", "invalid digit '9' in octal literal"},
+ {Int, "0o12a3", "0o12 a3", ""}, // only accept 0-9
+
+ // octal floats (invalid)
+ {Float, "0o.", "0o.", "invalid radix point in octal literal"},
+ {Float, "0o.2", "0o.2", "invalid radix point in octal literal"},
+ {Float, "0o1.2", "0o1.2", "invalid radix point in octal literal"},
+ {Float, "0o1E+2", "0o1E+2", "'E' exponent requires decimal mantissa"},
+ {Float, "0o1p10", "0o1p10", "'p' exponent requires hexadecimal mantissa"},
+
+ // 0-octals
+ {Int, "0", "0", ""},
+ {Int, "0123", "0123", ""},
+
+ {Int, "08123", "08123", "invalid digit '8' in octal literal"},
+ {Int, "01293", "01293", "invalid digit '9' in octal literal"},
+ {Int, "0F.", "0 F .", ""}, // only accept 0-9
+ {Int, "0123F.", "0123 F .", ""},
+ {Int, "0123456x", "0123456 x", ""},
+
+ // decimals
+ {Int, "1", "1", ""},
+ {Int, "1234", "1234", ""},
+
+ {Int, "1f", "1 f", ""}, // only accept 0-9
+
+ // decimal floats
+ {Float, "0.", "0.", ""},
+ {Float, "123.", "123.", ""},
+ {Float, "0123.", "0123.", ""},
+
+ {Float, ".0", ".0", ""},
+ {Float, ".123", ".123", ""},
+ {Float, ".0123", ".0123", ""},
+
+ {Float, "0.0", "0.0", ""},
+ {Float, "123.123", "123.123", ""},
+ {Float, "0123.0123", "0123.0123", ""},
+
+ {Float, "0e0", "0e0", ""},
+ {Float, "123e+0", "123e+0", ""},
+ {Float, "0123E-1", "0123E-1", ""},
+
+ {Float, "0.e+1", "0.e+1", ""},
+ {Float, "123.E-10", "123.E-10", ""},
+ {Float, "0123.e123", "0123.e123", ""},
+
+ {Float, ".0e-1", ".0e-1", ""},
+ {Float, ".123E+10", ".123E+10", ""},
+ {Float, ".0123E123", ".0123E123", ""},
+
+ {Float, "0.0e1", "0.0e1", ""},
+ {Float, "123.123E-10", "123.123E-10", ""},
+ {Float, "0123.0123e+456", "0123.0123e+456", ""},
+
+ {Float, "0e", "0e", "exponent has no digits"},
+ {Float, "0E+", "0E+", "exponent has no digits"},
+ {Float, "1e+f", "1e+ f", "exponent has no digits"},
+ {Float, "0p0", "0p0", "'p' exponent requires hexadecimal mantissa"},
+ {Float, "1.0P-1", "1.0P-1", "'P' exponent requires hexadecimal mantissa"},
+
+ // hexadecimals
+ {Int, "0x0", "0x0", ""},
+ {Int, "0x1234", "0x1234", ""},
+ {Int, "0xcafef00d", "0xcafef00d", ""},
+ {Int, "0XCAFEF00D", "0XCAFEF00D", ""},
+
+ {Int, "0x", "0x", "hexadecimal literal has no digits"},
+ {Int, "0x1g", "0x1 g", ""},
+
+ // hexadecimal floats
+ {Float, "0x0p0", "0x0p0", ""},
+ {Float, "0x12efp-123", "0x12efp-123", ""},
+ {Float, "0xABCD.p+0", "0xABCD.p+0", ""},
+ {Float, "0x.0189P-0", "0x.0189P-0", ""},
+ {Float, "0x1.ffffp+1023", "0x1.ffffp+1023", ""},
+
+ {Float, "0x.", "0x.", "hexadecimal literal has no digits"},
+ {Float, "0x0.", "0x0.", "hexadecimal mantissa requires a 'p' exponent"},
+ {Float, "0x.0", "0x.0", "hexadecimal mantissa requires a 'p' exponent"},
+ {Float, "0x1.1", "0x1.1", "hexadecimal mantissa requires a 'p' exponent"},
+ {Float, "0x1.1e0", "0x1.1e0", "hexadecimal mantissa requires a 'p' exponent"},
+ {Float, "0x1.2gp1a", "0x1.2 gp1a", "hexadecimal mantissa requires a 'p' exponent"},
+ {Float, "0x0p", "0x0p", "exponent has no digits"},
+ {Float, "0xeP-", "0xeP-", "exponent has no digits"},
+ {Float, "0x1234PAB", "0x1234P AB", "exponent has no digits"},
+ {Float, "0x1.2p1a", "0x1.2p1 a", ""},
+
+ // separators
+ {Int, "0b_1000_0001", "0b_1000_0001", ""},
+ {Int, "0o_600", "0o_600", ""},
+ {Int, "0_466", "0_466", ""},
+ {Int, "1_000", "1_000", ""},
+ {Float, "1_000.000_1", "1_000.000_1", ""},
+ {Int, "0x_f00d", "0x_f00d", ""},
+ {Float, "0x_f00d.0p1_2", "0x_f00d.0p1_2", ""},
+
+ {Int, "0b__1000", "0b__1000", "'_' must separate successive digits"},
+ {Int, "0o60___0", "0o60___0", "'_' must separate successive digits"},
+ {Int, "0466_", "0466_", "'_' must separate successive digits"},
+ {Float, "1_.", "1_.", "'_' must separate successive digits"},
+ {Float, "0._1", "0._1", "'_' must separate successive digits"},
+ {Float, "2.7_e0", "2.7_e0", "'_' must separate successive digits"},
+ {Int, "0x___0", "0x___0", "'_' must separate successive digits"},
+ {Float, "0x1.0_p0", "0x1.0_p0", "'_' must separate successive digits"},
+ } {
+ s := new(Scanner).Init(strings.NewReader(test.src))
+ var err string
+ s.Error = func(s *Scanner, msg string) {
+ if err == "" {
+ err = msg
+ }
+ }
+
+ for i, want := range strings.Split(test.tokens, " ") {
+ err = ""
+ tok := s.Scan()
+ lit := s.TokenText()
+ if i == 0 {
+ if tok != test.tok {
+ t.Errorf("%q: got token %s; want %s", test.src, TokenString(tok), TokenString(test.tok))
+ }
+ if err != test.err {
+ t.Errorf("%q: got error %q; want %q", test.src, err, test.err)
+ }
+ }
+ if lit != want {
+ t.Errorf("%q: got literal %q (%s); want %s", test.src, lit, TokenString(tok), want)
+ }
+ }
+
+ // make sure we read all
+ if tok := s.Scan(); tok != EOF {
+ t.Errorf("%q: got %s; want EOF", test.src, TokenString(tok))
+ }
+ }
+}
