| // Copyright 2016 The Go Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style | |
| // license that can be found in the LICENSE file. | |
| // This file implements scanner, a lexical tokenizer for | |
| // Go source. After initialization, consecutive calls of | |
| // next advance the scanner one token at a time. | |
| // | |
| // This file, source.go, and tokens.go are self-contained | |
| // (go tool compile scanner.go source.go tokens.go compiles) | |
| // and thus could be made into its own package. | |
| package syntax | |
| import ( | |
| "fmt" | |
| "io" | |
| "unicode" | |
| "unicode/utf8" | |
| ) | |
| // The mode flags below control which comments are reported | |
| // by calling the error handler. If no flag is set, comments | |
| // are ignored. | |
| const ( | |
| comments uint = 1 << iota // call handler for all comments | |
| directives // call handler for directives only | |
| ) | |
| type scanner struct { | |
| source | |
| mode uint | |
| nlsemi bool // if set '\n' and EOF translate to ';' | |
| // current token, valid after calling next() | |
| line, col uint | |
| tok token | |
| lit string // valid if tok is _Name, _Literal, or _Semi ("semicolon", "newline", or "EOF") | |
| kind LitKind // valid if tok is _Literal | |
| op Operator // valid if tok is _Operator, _AssignOp, or _IncOp | |
| prec int // valid if tok is _Operator, _AssignOp, or _IncOp | |
| } | |
| func (s *scanner) init(src io.Reader, errh func(line, col uint, msg string), mode uint) { | |
| s.source.init(src, errh) | |
| s.mode = mode | |
| s.nlsemi = false | |
| } | |
| // next advances the scanner by reading the next token. | |
| // | |
| // If a read, source encoding, or lexical error occurs, next calls | |
| // the installed error handler with the respective error position | |
| // and message. The error message is guaranteed to be non-empty and | |
| // never starts with a '/'. The error handler must exist. | |
| // | |
| // If the scanner mode includes the comments flag and a comment | |
| // (including comments containing directives) is encountered, the | |
| // error handler is also called with each comment position and text | |
| // (including opening /* or // and closing */, but without a newline | |
| // at the end of line comments). Comment text always starts with a / | |
| // which can be used to distinguish these handler calls from errors. | |
| // | |
| // If the scanner mode includes the directives (but not the comments) | |
| // flag, only comments containing a //line, /*line, or //go: directive | |
| // are reported, in the same way as regular comments. Directives in | |
| // //-style comments are only recognized if they are at the beginning | |
| // of a line. | |
| // | |
| func (s *scanner) next() { | |
| nlsemi := s.nlsemi | |
| s.nlsemi = false | |
| redo: | |
| // skip white space | |
| c := s.getr() | |
| for c == ' ' || c == '\t' || c == '\n' && !nlsemi || c == '\r' { | |
| c = s.getr() | |
| } | |
| // token start | |
| s.line, s.col = s.source.line0, s.source.col0 | |
| if isLetter(c) || c >= utf8.RuneSelf && s.isIdentRune(c, true) { | |
| s.ident() | |
| return | |
| } | |
| switch c { | |
| case -1: | |
| if nlsemi { | |
| s.lit = "EOF" | |
| s.tok = _Semi | |
| break | |
| } | |
| s.tok = _EOF | |
| case '\n': | |
| s.lit = "newline" | |
| s.tok = _Semi | |
| case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': | |
| s.number(c) | |
| case '"': | |
| s.stdString() | |
| case '`': | |
| s.rawString() | |
| case '\'': | |
| s.rune() | |
| case '(': | |
| s.tok = _Lparen | |
| case '[': | |
| s.tok = _Lbrack | |
| case '{': | |
| s.tok = _Lbrace | |
| case ',': | |
| s.tok = _Comma | |
| case ';': | |
| s.lit = "semicolon" | |
| s.tok = _Semi | |
| case ')': | |
| s.nlsemi = true | |
| s.tok = _Rparen | |
| case ']': | |
| s.nlsemi = true | |
| s.tok = _Rbrack | |
| case '}': | |
| s.nlsemi = true | |
| s.tok = _Rbrace | |
| case ':': | |
| if s.getr() == '=' { | |
| s.tok = _Define | |
| break | |
| } | |
| s.ungetr() | |
| s.tok = _Colon | |
| case '.': | |
| c = s.getr() | |
| if isDigit(c) { | |
| s.ungetr2() | |
| s.number('.') | |
| break | |
| } | |
| if c == '.' { | |
| c = s.getr() | |
| if c == '.' { | |
| s.tok = _DotDotDot | |
| break | |
| } | |
| s.ungetr2() | |
| } | |
| s.ungetr() | |
| s.tok = _Dot | |
| case '+': | |
| s.op, s.prec = Add, precAdd | |
| c = s.getr() | |
| if c != '+' { | |
| goto assignop | |
| } | |
| s.nlsemi = true | |
| s.tok = _IncOp | |
| case '-': | |
| s.op, s.prec = Sub, precAdd | |
| c = s.getr() | |
| if c != '-' { | |
| goto assignop | |
| } | |
| s.nlsemi = true | |
| s.tok = _IncOp | |
| case '*': | |
| s.op, s.prec = Mul, precMul | |
| // don't goto assignop - want _Star token | |
| if s.getr() == '=' { | |
| s.tok = _AssignOp | |
| break | |
| } | |
| s.ungetr() | |
| s.tok = _Star | |
| case '/': | |
| c = s.getr() | |
| if c == '/' { | |
| s.lineComment() | |
| goto redo | |
| } | |
| if c == '*' { | |
| s.fullComment() | |
| if s.source.line > s.line && nlsemi { | |
| // A multi-line comment acts like a newline; | |
| // it translates to a ';' if nlsemi is set. | |
| s.lit = "newline" | |
| s.tok = _Semi | |
| break | |
| } | |
| goto redo | |
| } | |
| s.op, s.prec = Div, precMul | |
| goto assignop | |
| case '%': | |
| s.op, s.prec = Rem, precMul | |
| c = s.getr() | |
| goto assignop | |
| case '&': | |
| c = s.getr() | |
| if c == '&' { | |
| s.op, s.prec = AndAnd, precAndAnd | |
| s.tok = _Operator | |
| break | |
| } | |
| s.op, s.prec = And, precMul | |
| if c == '^' { | |
| s.op = AndNot | |
| c = s.getr() | |
| } | |
| goto assignop | |
| case '|': | |
| c = s.getr() | |
| if c == '|' { | |
| s.op, s.prec = OrOr, precOrOr | |
| s.tok = _Operator | |
| break | |
| } | |
| s.op, s.prec = Or, precAdd | |
| goto assignop | |
| case '^': | |
| s.op, s.prec = Xor, precAdd | |
| c = s.getr() | |
| goto assignop | |
| case '<': | |
| c = s.getr() | |
| if c == '=' { | |
| s.op, s.prec = Leq, precCmp | |
| s.tok = _Operator | |
| break | |
| } | |
| if c == '<' { | |
| s.op, s.prec = Shl, precMul | |
| c = s.getr() | |
| goto assignop | |
| } | |
| if c == '-' { | |
| s.tok = _Arrow | |
| break | |
| } | |
| s.ungetr() | |
| s.op, s.prec = Lss, precCmp | |
| s.tok = _Operator | |
| case '>': | |
| c = s.getr() | |
| if c == '=' { | |
| s.op, s.prec = Geq, precCmp | |
| s.tok = _Operator | |
| break | |
| } | |
| if c == '>' { | |
| s.op, s.prec = Shr, precMul | |
| c = s.getr() | |
| goto assignop | |
| } | |
| s.ungetr() | |
| s.op, s.prec = Gtr, precCmp | |
| s.tok = _Operator | |
| case '=': | |
| if s.getr() == '=' { | |
| s.op, s.prec = Eql, precCmp | |
| s.tok = _Operator | |
| break | |
| } | |
| s.ungetr() | |
| s.tok = _Assign | |
| case '!': | |
| if s.getr() == '=' { | |
| s.op, s.prec = Neq, precCmp | |
| s.tok = _Operator | |
| break | |
| } | |
| s.ungetr() | |
| s.op, s.prec = Not, 0 | |
| s.tok = _Operator | |
| default: | |
| s.tok = 0 | |
| s.error(fmt.Sprintf("invalid character %#U", c)) | |
| goto redo | |
| } | |
| return | |
| assignop: | |
| if c == '=' { | |
| s.tok = _AssignOp | |
| return | |
| } | |
| s.ungetr() | |
| s.tok = _Operator | |
| } | |
| func isLetter(c rune) bool { | |
| return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '_' | |
| } | |
| func isDigit(c rune) bool { | |
| return '0' <= c && c <= '9' | |
| } | |
| func (s *scanner) ident() { | |
| s.startLit() | |
| // accelerate common case (7bit ASCII) | |
| c := s.getr() | |
| for isLetter(c) || isDigit(c) { | |
| c = s.getr() | |
| } | |
| // general case | |
| if c >= utf8.RuneSelf { | |
| for s.isIdentRune(c, false) { | |
| c = s.getr() | |
| } | |
| } | |
| s.ungetr() | |
| lit := s.stopLit() | |
| // possibly a keyword | |
| if len(lit) >= 2 { | |
| if tok := keywordMap[hash(lit)]; tok != 0 && tokStrFast(tok) == string(lit) { | |
| s.nlsemi = contains(1<<_Break|1<<_Continue|1<<_Fallthrough|1<<_Return, tok) | |
| s.tok = tok | |
| return | |
| } | |
| } | |
| s.nlsemi = true | |
| s.lit = string(lit) | |
| s.tok = _Name | |
| } | |
| // tokStrFast is a faster version of token.String, which assumes that tok | |
| // is one of the valid tokens - and can thus skip bounds checks. | |
| func tokStrFast(tok token) string { | |
| return _token_name[_token_index[tok-1]:_token_index[tok]] | |
| } | |
| func (s *scanner) isIdentRune(c rune, first bool) bool { | |
| switch { | |
| case unicode.IsLetter(c) || c == '_': | |
| // ok | |
| case unicode.IsDigit(c): | |
| if first { | |
| s.error(fmt.Sprintf("identifier cannot begin with digit %#U", c)) | |
| } | |
| case c >= utf8.RuneSelf: | |
| s.error(fmt.Sprintf("invalid identifier character %#U", c)) | |
| default: | |
| return false | |
| } | |
| return true | |
| } | |
| // hash is a perfect hash function for keywords. | |
| // It assumes that s has at least length 2. | |
| func hash(s []byte) uint { | |
| return (uint(s[0])<<4 ^ uint(s[1]) + uint(len(s))) & uint(len(keywordMap)-1) | |
| } | |
| var keywordMap [1 << 6]token // size must be power of two | |
| func init() { | |
| // populate keywordMap | |
| for tok := _Break; tok <= _Var; tok++ { | |
| h := hash([]byte(tok.String())) | |
| if keywordMap[h] != 0 { | |
| panic("imperfect hash") | |
| } | |
| keywordMap[h] = tok | |
| } | |
| } | |
| func (s *scanner) number(c rune) { | |
| s.startLit() | |
| if c != '.' { | |
| s.kind = IntLit // until proven otherwise | |
| if c == '0' { | |
| c = s.getr() | |
| if c == 'x' || c == 'X' { | |
| // hex | |
| c = s.getr() | |
| hasDigit := false | |
| for isDigit(c) || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { | |
| c = s.getr() | |
| hasDigit = true | |
| } | |
| if !hasDigit { | |
| s.error("malformed hex constant") | |
| } | |
| goto done | |
| } | |
| // decimal 0, octal, or float | |
| has8or9 := false | |
| for isDigit(c) { | |
| if c > '7' { | |
| has8or9 = true | |
| } | |
| c = s.getr() | |
| } | |
| if c != '.' && c != 'e' && c != 'E' && c != 'i' { | |
| // octal | |
| if has8or9 { | |
| s.error("malformed octal constant") | |
| } | |
| goto done | |
| } | |
| } else { | |
| // decimal or float | |
| for isDigit(c) { | |
| c = s.getr() | |
| } | |
| } | |
| } | |
| // float | |
| if c == '.' { | |
| s.kind = FloatLit | |
| c = s.getr() | |
| for isDigit(c) { | |
| c = s.getr() | |
| } | |
| } | |
| // exponent | |
| if c == 'e' || c == 'E' { | |
| s.kind = FloatLit | |
| c = s.getr() | |
| if c == '-' || c == '+' { | |
| c = s.getr() | |
| } | |
| if !isDigit(c) { | |
| s.error("malformed floating-point constant exponent") | |
| } | |
| for isDigit(c) { | |
| c = s.getr() | |
| } | |
| } | |
| // complex | |
| if c == 'i' { | |
| s.kind = ImagLit | |
| s.getr() | |
| } | |
| done: | |
| s.ungetr() | |
| s.nlsemi = true | |
| s.lit = string(s.stopLit()) | |
| s.tok = _Literal | |
| } | |
| func (s *scanner) rune() { | |
| s.startLit() | |
| ok := true // only report errors if we're ok so far | |
| n := 0 | |
| for ; ; n++ { | |
| r := s.getr() | |
| if r == '\'' { | |
| break | |
| } | |
| if r == '\\' { | |
| if !s.escape('\'') { | |
| ok = false | |
| } | |
| continue | |
| } | |
| if r == '\n' { | |
| s.ungetr() // assume newline is not part of literal | |
| if ok { | |
| s.error("newline in character literal") | |
| ok = false | |
| } | |
| break | |
| } | |
| if r < 0 { | |
| if ok { | |
| s.errh(s.line, s.col, "invalid character literal (missing closing ')") | |
| ok = false | |
| } | |
| break | |
| } | |
| } | |
| if ok { | |
| if n == 0 { | |
| s.error("empty character literal or unescaped ' in character literal") | |
| } else if n != 1 { | |
| s.errh(s.line, s.col, "invalid character literal (more than one character)") | |
| } | |
| } | |
| s.nlsemi = true | |
| s.lit = string(s.stopLit()) | |
| s.kind = RuneLit | |
| s.tok = _Literal | |
| } | |
| func (s *scanner) stdString() { | |
| s.startLit() | |
| for { | |
| r := s.getr() | |
| if r == '"' { | |
| break | |
| } | |
| if r == '\\' { | |
| s.escape('"') | |
| continue | |
| } | |
| if r == '\n' { | |
| s.ungetr() // assume newline is not part of literal | |
| s.error("newline in string") | |
| break | |
| } | |
| if r < 0 { | |
| s.errh(s.line, s.col, "string not terminated") | |
| break | |
| } | |
| } | |
| s.nlsemi = true | |
| s.lit = string(s.stopLit()) | |
| s.kind = StringLit | |
| s.tok = _Literal | |
| } | |
| func (s *scanner) rawString() { | |
| s.startLit() | |
| for { | |
| r := s.getr() | |
| if r == '`' { | |
| break | |
| } | |
| if r < 0 { | |
| s.errh(s.line, s.col, "string not terminated") | |
| break | |
| } | |
| } | |
| // We leave CRs in the string since they are part of the | |
| // literal (even though they are not part of the literal | |
| // value). | |
| s.nlsemi = true | |
| s.lit = string(s.stopLit()) | |
| s.kind = StringLit | |
| s.tok = _Literal | |
| } | |
| func (s *scanner) comment(text string) { | |
| s.errh(s.line, s.col, text) | |
| } | |
| func (s *scanner) skipLine(r rune) { | |
| for r >= 0 { | |
| if r == '\n' { | |
| s.ungetr() // don't consume '\n' - needed for nlsemi logic | |
| break | |
| } | |
| r = s.getr() | |
| } | |
| } | |
| func (s *scanner) lineComment() { | |
| r := s.getr() | |
| if s.mode&comments != 0 { | |
| s.startLit() | |
| s.skipLine(r) | |
| s.comment("//" + string(s.stopLit())) | |
| return | |
| } | |
| // directives must start at the beginning of the line (s.col == colbase) | |
| if s.mode&directives == 0 || s.col != colbase || (r != 'g' && r != 'l') { | |
| s.skipLine(r) | |
| return | |
| } | |
| // recognize go: or line directives | |
| prefix := "go:" | |
| if r == 'l' { | |
| prefix = "line " | |
| } | |
| for _, m := range prefix { | |
| if r != m { | |
| s.skipLine(r) | |
| return | |
| } | |
| r = s.getr() | |
| } | |
| // directive text | |
| s.startLit() | |
| s.skipLine(r) | |
| s.comment("//" + prefix + string(s.stopLit())) | |
| } | |
| func (s *scanner) skipComment(r rune) bool { | |
| for r >= 0 { | |
| for r == '*' { | |
| r = s.getr() | |
| if r == '/' { | |
| return true | |
| } | |
| } | |
| r = s.getr() | |
| } | |
| s.errh(s.line, s.col, "comment not terminated") | |
| return false | |
| } | |
| func (s *scanner) fullComment() { | |
| r := s.getr() | |
| if s.mode&comments != 0 { | |
| s.startLit() | |
| if s.skipComment(r) { | |
| s.comment("/*" + string(s.stopLit())) | |
| } else { | |
| s.killLit() // not a complete comment - ignore | |
| } | |
| return | |
| } | |
| if s.mode&directives == 0 || r != 'l' { | |
| s.skipComment(r) | |
| return | |
| } | |
| // recognize line directive | |
| const prefix = "line " | |
| for _, m := range prefix { | |
| if r != m { | |
| s.skipComment(r) | |
| return | |
| } | |
| r = s.getr() | |
| } | |
| // directive text | |
| s.startLit() | |
| if s.skipComment(r) { | |
| s.comment("/*" + prefix + string(s.stopLit())) | |
| } else { | |
| s.killLit() // not a complete comment - ignore | |
| } | |
| } | |
| func (s *scanner) escape(quote rune) bool { | |
| var n int | |
| var base, max uint32 | |
| c := s.getr() | |
| switch c { | |
| case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote: | |
| return true | |
| case '0', '1', '2', '3', '4', '5', '6', '7': | |
| n, base, max = 3, 8, 255 | |
| case 'x': | |
| c = s.getr() | |
| n, base, max = 2, 16, 255 | |
| case 'u': | |
| c = s.getr() | |
| n, base, max = 4, 16, unicode.MaxRune | |
| case 'U': | |
| c = s.getr() | |
| n, base, max = 8, 16, unicode.MaxRune | |
| default: | |
| if c < 0 { | |
| return true // complain in caller about EOF | |
| } | |
| s.error("unknown escape sequence") | |
| return false | |
| } | |
| var x uint32 | |
| for i := n; i > 0; i-- { | |
| d := base | |
| switch { | |
| case isDigit(c): | |
| d = uint32(c) - '0' | |
| case 'a' <= c && c <= 'f': | |
| d = uint32(c) - ('a' - 10) | |
| case 'A' <= c && c <= 'F': | |
| d = uint32(c) - ('A' - 10) | |
| } | |
| if d >= base { | |
| if c < 0 { | |
| return true // complain in caller about EOF | |
| } | |
| kind := "hex" | |
| if base == 8 { | |
| kind = "octal" | |
| } | |
| s.error(fmt.Sprintf("non-%s character in escape sequence: %c", kind, c)) | |
| s.ungetr() | |
| return false | |
| } | |
| // d < base | |
| x = x*base + d | |
| c = s.getr() | |
| } | |
| s.ungetr() | |
| if x > max && base == 8 { | |
| s.error(fmt.Sprintf("octal escape value > 255: %d", x)) | |
| return false | |
| } | |
| if x > max || 0xD800 <= x && x < 0xE000 /* surrogate range */ { | |
| s.error("escape sequence is invalid Unicode code point") | |
| return false | |
| } | |
| return true | |
| } |