blob: b84fcc5fd16d8ad27369f752f89b2b6a91314ad1 [file] [log] [blame]
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package syntax
import (
"fmt"
"io"
"strings"
"unicode"
"unicode/utf8"
)
type scanner struct {
source
nlsemi bool // if set '\n' and EOF translate to ';'
pragma Pragma
// current token, valid after calling next()
pos, line int
tok token
lit string // valid if tok is _Name or _Literal
kind LitKind // valid if tok is _Literal
op Operator // valid if tok is _Operator, _AssignOp, or _IncOp
prec int // valid if tok is _Operator, _AssignOp, or _IncOp
pragh PragmaHandler
}
func (s *scanner) init(src io.Reader, errh ErrorHandler, pragh PragmaHandler) {
s.source.init(src, errh)
s.nlsemi = false
s.pragh = pragh
}
func (s *scanner) next() {
nlsemi := s.nlsemi
s.nlsemi = false
redo:
// skip white space
c := s.getr()
for c == ' ' || c == '\t' || c == '\n' && !nlsemi || c == '\r' {
c = s.getr()
}
// token start
s.pos, s.line = s.source.pos0(), s.source.line0
if isLetter(c) || c >= utf8.RuneSelf && (unicode.IsLetter(c) || s.isCompatRune(c, true)) {
s.ident()
return
}
switch c {
case -1:
if nlsemi {
s.tok = _Semi
break
}
s.tok = _EOF
case '\n':
s.tok = _Semi
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
s.number(c)
case '"':
s.stdString()
case '`':
s.rawString()
case '\'':
s.rune()
case '(':
s.tok = _Lparen
case '[':
s.tok = _Lbrack
case '{':
s.tok = _Lbrace
case ',':
s.tok = _Comma
case ';':
s.tok = _Semi
case ')':
s.nlsemi = true
s.tok = _Rparen
case ']':
s.nlsemi = true
s.tok = _Rbrack
case '}':
s.nlsemi = true
s.tok = _Rbrace
case ':':
if s.getr() == '=' {
s.tok = _Define
break
}
s.ungetr()
s.tok = _Colon
case '.':
c = s.getr()
if isDigit(c) {
s.ungetr()
s.source.r0-- // make sure '.' is part of literal (line cannot have changed)
s.number('.')
break
}
if c == '.' {
c = s.getr()
if c == '.' {
s.tok = _DotDotDot
break
}
s.ungetr()
s.source.r0-- // make next ungetr work (line cannot have changed)
}
s.ungetr()
s.tok = _Dot
case '+':
s.op, s.prec = Add, precAdd
c = s.getr()
if c != '+' {
goto assignop
}
s.nlsemi = true
s.tok = _IncOp
case '-':
s.op, s.prec = Sub, precAdd
c = s.getr()
if c != '-' {
goto assignop
}
s.nlsemi = true
s.tok = _IncOp
case '*':
s.op, s.prec = Mul, precMul
// don't goto assignop - want _Star token
if s.getr() == '=' {
s.tok = _AssignOp
break
}
s.ungetr()
s.tok = _Star
case '/':
c = s.getr()
if c == '/' {
s.lineComment()
goto redo
}
if c == '*' {
s.fullComment()
if s.source.line > s.line && nlsemi {
// A multi-line comment acts like a newline;
// it translates to a ';' if nlsemi is set.
s.tok = _Semi
break
}
goto redo
}
s.op, s.prec = Div, precMul
goto assignop
case '%':
s.op, s.prec = Rem, precMul
c = s.getr()
goto assignop
case '&':
c = s.getr()
if c == '&' {
s.op, s.prec = AndAnd, precAndAnd
s.tok = _Operator
break
}
s.op, s.prec = And, precMul
if c == '^' {
s.op = AndNot
c = s.getr()
}
goto assignop
case '|':
c = s.getr()
if c == '|' {
s.op, s.prec = OrOr, precOrOr
s.tok = _Operator
break
}
s.op, s.prec = Or, precAdd
goto assignop
case '~':
s.error("bitwise complement operator is ^")
fallthrough
case '^':
s.op, s.prec = Xor, precAdd
c = s.getr()
goto assignop
case '<':
c = s.getr()
if c == '=' {
s.op, s.prec = Leq, precCmp
s.tok = _Operator
break
}
if c == '<' {
s.op, s.prec = Shl, precMul
c = s.getr()
goto assignop
}
if c == '-' {
s.tok = _Arrow
break
}
s.ungetr()
s.op, s.prec = Lss, precCmp
s.tok = _Operator
case '>':
c = s.getr()
if c == '=' {
s.op, s.prec = Geq, precCmp
s.tok = _Operator
break
}
if c == '>' {
s.op, s.prec = Shr, precMul
c = s.getr()
goto assignop
}
s.ungetr()
s.op, s.prec = Gtr, precCmp
s.tok = _Operator
case '=':
if s.getr() == '=' {
s.op, s.prec = Eql, precCmp
s.tok = _Operator
break
}
s.ungetr()
s.tok = _Assign
case '!':
if s.getr() == '=' {
s.op, s.prec = Neq, precCmp
s.tok = _Operator
break
}
s.ungetr()
s.op, s.prec = Not, 0
s.tok = _Operator
default:
s.tok = 0
s.error(fmt.Sprintf("illegal character %#U", c))
goto redo
}
return
assignop:
if c == '=' {
s.tok = _AssignOp
return
}
s.ungetr()
s.tok = _Operator
}
func isLetter(c rune) bool {
return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '_'
}
func isDigit(c rune) bool {
return '0' <= c && c <= '9'
}
func (s *scanner) ident() {
s.startLit()
// accelerate common case (7bit ASCII)
c := s.getr()
for isLetter(c) || isDigit(c) {
c = s.getr()
}
// general case
if c >= utf8.RuneSelf {
for unicode.IsLetter(c) || c == '_' || unicode.IsDigit(c) || s.isCompatRune(c, false) {
c = s.getr()
}
}
s.ungetr()
lit := s.stopLit()
// possibly a keyword
if len(lit) >= 2 {
if tok := keywordMap[hash(lit)]; tok != 0 && tokstrings[tok] == string(lit) {
s.nlsemi = contains(1<<_Break|1<<_Continue|1<<_Fallthrough|1<<_Return, tok)
s.tok = tok
return
}
}
s.nlsemi = true
s.lit = string(lit)
s.tok = _Name
}
func (s *scanner) isCompatRune(c rune, start bool) bool {
if !gcCompat || c < utf8.RuneSelf {
return false
}
if start && unicode.IsNumber(c) {
s.error(fmt.Sprintf("identifier cannot begin with digit %#U", c))
} else {
s.error(fmt.Sprintf("invalid identifier character %#U", c))
}
return true
}
// hash is a perfect hash function for keywords.
// It assumes that s has at least length 2.
func hash(s []byte) uint {
return (uint(s[0])<<4 ^ uint(s[1]) + uint(len(s))) & uint(len(keywordMap)-1)
}
var keywordMap [1 << 6]token // size must be power of two
func init() {
// populate keywordMap
for tok := _Break; tok <= _Var; tok++ {
h := hash([]byte(tokstrings[tok]))
if keywordMap[h] != 0 {
panic("imperfect hash")
}
keywordMap[h] = tok
}
}
func (s *scanner) number(c rune) {
s.startLit()
if c != '.' {
s.kind = IntLit // until proven otherwise
if c == '0' {
c = s.getr()
if c == 'x' || c == 'X' {
// hex
c = s.getr()
hasDigit := false
for isDigit(c) || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
c = s.getr()
hasDigit = true
}
if !hasDigit {
s.error("malformed hex constant")
}
goto done
}
// decimal 0, octal, or float
has8or9 := false
for isDigit(c) {
if c > '7' {
has8or9 = true
}
c = s.getr()
}
if c != '.' && c != 'e' && c != 'E' && c != 'i' {
// octal
if has8or9 {
s.error("malformed octal constant")
}
goto done
}
} else {
// decimal or float
for isDigit(c) {
c = s.getr()
}
}
}
// float
if c == '.' {
s.kind = FloatLit
c = s.getr()
for isDigit(c) {
c = s.getr()
}
}
// exponent
if c == 'e' || c == 'E' {
s.kind = FloatLit
c = s.getr()
if c == '-' || c == '+' {
c = s.getr()
}
if !isDigit(c) {
s.error("malformed floating-point constant exponent")
}
for isDigit(c) {
c = s.getr()
}
}
// complex
if c == 'i' {
s.kind = ImagLit
s.getr()
}
done:
s.ungetr()
s.nlsemi = true
s.lit = string(s.stopLit())
s.tok = _Literal
}
func (s *scanner) stdString() {
s.startLit()
for {
r := s.getr()
if r == '"' {
break
}
if r == '\\' {
s.escape('"')
continue
}
if r == '\n' {
s.ungetr() // assume newline is not part of literal
s.error("newline in string")
break
}
if r < 0 {
s.error_at(s.pos, s.line, "string not terminated")
break
}
}
s.nlsemi = true
s.lit = string(s.stopLit())
s.kind = StringLit
s.tok = _Literal
}
func (s *scanner) rawString() {
s.startLit()
for {
r := s.getr()
if r == '`' {
break
}
if r < 0 {
s.error_at(s.pos, s.line, "string not terminated")
break
}
}
// We leave CRs in the string since they are part of the
// literal (even though they are not part of the literal
// value).
s.nlsemi = true
s.lit = string(s.stopLit())
s.kind = StringLit
s.tok = _Literal
}
func (s *scanner) rune() {
s.startLit()
r := s.getr()
ok := false
if r == '\'' {
s.error("empty character literal or unescaped ' in character literal")
} else if r == '\n' {
s.ungetr() // assume newline is not part of literal
s.error("newline in character literal")
} else {
ok = true
if r == '\\' {
ok = s.escape('\'')
}
}
r = s.getr()
if r != '\'' {
// only report error if we're ok so far
if ok {
s.error("missing '")
}
s.ungetr()
}
s.nlsemi = true
s.lit = string(s.stopLit())
s.kind = RuneLit
s.tok = _Literal
}
func (s *scanner) lineComment() {
// recognize pragmas
var prefix string
r := s.getr()
if s.pragh == nil {
goto skip
}
switch r {
case 'g':
prefix = "go:"
case 'l':
prefix = "line "
default:
goto skip
}
s.startLit()
for _, m := range prefix {
if r != m {
s.stopLit()
goto skip
}
r = s.getr()
}
for r >= 0 {
if r == '\n' {
s.ungetr()
break
}
r = s.getr()
}
s.pragma |= s.pragh(0, s.line, strings.TrimSuffix(string(s.stopLit()), "\r"))
return
skip:
// consume line
for r != '\n' && r >= 0 {
r = s.getr()
}
s.ungetr() // don't consume '\n' - needed for nlsemi logic
}
func (s *scanner) fullComment() {
for {
r := s.getr()
for r == '*' {
r = s.getr()
if r == '/' {
return
}
}
if r < 0 {
s.error_at(s.pos, s.line, "comment not terminated")
return
}
}
}
func (s *scanner) escape(quote rune) bool {
var n int
var base, max uint32
c := s.getr()
switch c {
case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote:
return true
case '0', '1', '2', '3', '4', '5', '6', '7':
n, base, max = 3, 8, 255
case 'x':
c = s.getr()
n, base, max = 2, 16, 255
case 'u':
c = s.getr()
n, base, max = 4, 16, unicode.MaxRune
case 'U':
c = s.getr()
n, base, max = 8, 16, unicode.MaxRune
default:
if c < 0 {
return true // complain in caller about EOF
}
s.error("unknown escape sequence")
return false
}
var x uint32
for i := n; i > 0; i-- {
d := base
switch {
case isDigit(c):
d = uint32(c) - '0'
case 'a' <= c && c <= 'f':
d = uint32(c) - ('a' - 10)
case 'A' <= c && c <= 'F':
d = uint32(c) - ('A' - 10)
}
if d >= base {
if c < 0 {
return true // complain in caller about EOF
}
if gcCompat {
name := "hex"
if base == 8 {
name = "octal"
}
s.error(fmt.Sprintf("non-%s character in escape sequence: %c", name, c))
} else {
if c != quote {
s.error(fmt.Sprintf("illegal character %#U in escape sequence", c))
} else {
s.error("escape sequence incomplete")
}
}
s.ungetr()
return false
}
// d < base
x = x*base + d
c = s.getr()
}
s.ungetr()
if x > max && base == 8 {
s.error(fmt.Sprintf("octal escape value > 255: %d", x))
return false
}
if x > max || 0xD800 <= x && x < 0xE000 /* surrogate range */ {
s.error("escape sequence is invalid Unicode code point")
return false
}
return true
}