| // Copyright 2009 The Go Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style | |
| // license that can be found in the LICENSE file. | |
| // Package token defines constants representing the lexical tokens of the Go | |
| // programming language and basic operations on tokens (printing, predicates). | |
| // | |
| package token | |
| import ( | |
| "strconv" | |
| "unicode" | |
| "unicode/utf8" | |
| ) | |
| // Token is the set of lexical tokens of the Go programming language. | |
| type Token int | |
| // The list of tokens. | |
| const ( | |
| // Special tokens | |
| ILLEGAL Token = iota | |
| EOF | |
| COMMENT | |
| literal_beg | |
| // Identifiers and basic type literals | |
| // (these tokens stand for classes of literals) | |
| IDENT // main | |
| INT // 12345 | |
| FLOAT // 123.45 | |
| IMAG // 123.45i | |
| CHAR // 'a' | |
| STRING // "abc" | |
| literal_end | |
| operator_beg | |
| // Operators and delimiters | |
| ADD // + | |
| SUB // - | |
| MUL // * | |
| QUO // / | |
| REM // % | |
| AND // & | |
| OR // | | |
| XOR // ^ | |
| SHL // << | |
| SHR // >> | |
| AND_NOT // &^ | |
| ADD_ASSIGN // += | |
| SUB_ASSIGN // -= | |
| MUL_ASSIGN // *= | |
| QUO_ASSIGN // /= | |
| REM_ASSIGN // %= | |
| AND_ASSIGN // &= | |
| OR_ASSIGN // |= | |
| XOR_ASSIGN // ^= | |
| SHL_ASSIGN // <<= | |
| SHR_ASSIGN // >>= | |
| AND_NOT_ASSIGN // &^= | |
| LAND // && | |
| LOR // || | |
| ARROW // <- | |
| INC // ++ | |
| DEC // -- | |
| EQL // == | |
| LSS // < | |
| GTR // > | |
| ASSIGN // = | |
| NOT // ! | |
| NEQ // != | |
| LEQ // <= | |
| GEQ // >= | |
| DEFINE // := | |
| ELLIPSIS // ... | |
| LPAREN // ( | |
| LBRACK // [ | |
| LBRACE // { | |
| COMMA // , | |
| PERIOD // . | |
| RPAREN // ) | |
| RBRACK // ] | |
| RBRACE // } | |
| SEMICOLON // ; | |
| COLON // : | |
| operator_end | |
| keyword_beg | |
| // Keywords | |
| BREAK | |
| CASE | |
| CHAN | |
| CONST | |
| CONTINUE | |
| DEFAULT | |
| DEFER | |
| ELSE | |
| FALLTHROUGH | |
| FOR | |
| FUNC | |
| GO | |
| GOTO | |
| IF | |
| IMPORT | |
| INTERFACE | |
| MAP | |
| PACKAGE | |
| RANGE | |
| RETURN | |
| SELECT | |
| STRUCT | |
| SWITCH | |
| TYPE | |
| VAR | |
| keyword_end | |
| ) | |
| var tokens = [...]string{ | |
| ILLEGAL: "ILLEGAL", | |
| EOF: "EOF", | |
| COMMENT: "COMMENT", | |
| IDENT: "IDENT", | |
| INT: "INT", | |
| FLOAT: "FLOAT", | |
| IMAG: "IMAG", | |
| CHAR: "CHAR", | |
| STRING: "STRING", | |
| ADD: "+", | |
| SUB: "-", | |
| MUL: "*", | |
| QUO: "/", | |
| REM: "%", | |
| AND: "&", | |
| OR: "|", | |
| XOR: "^", | |
| SHL: "<<", | |
| SHR: ">>", | |
| AND_NOT: "&^", | |
| ADD_ASSIGN: "+=", | |
| SUB_ASSIGN: "-=", | |
| MUL_ASSIGN: "*=", | |
| QUO_ASSIGN: "/=", | |
| REM_ASSIGN: "%=", | |
| AND_ASSIGN: "&=", | |
| OR_ASSIGN: "|=", | |
| XOR_ASSIGN: "^=", | |
| SHL_ASSIGN: "<<=", | |
| SHR_ASSIGN: ">>=", | |
| AND_NOT_ASSIGN: "&^=", | |
| LAND: "&&", | |
| LOR: "||", | |
| ARROW: "<-", | |
| INC: "++", | |
| DEC: "--", | |
| EQL: "==", | |
| LSS: "<", | |
| GTR: ">", | |
| ASSIGN: "=", | |
| NOT: "!", | |
| NEQ: "!=", | |
| LEQ: "<=", | |
| GEQ: ">=", | |
| DEFINE: ":=", | |
| ELLIPSIS: "...", | |
| LPAREN: "(", | |
| LBRACK: "[", | |
| LBRACE: "{", | |
| COMMA: ",", | |
| PERIOD: ".", | |
| RPAREN: ")", | |
| RBRACK: "]", | |
| RBRACE: "}", | |
| SEMICOLON: ";", | |
| COLON: ":", | |
| BREAK: "break", | |
| CASE: "case", | |
| CHAN: "chan", | |
| CONST: "const", | |
| CONTINUE: "continue", | |
| DEFAULT: "default", | |
| DEFER: "defer", | |
| ELSE: "else", | |
| FALLTHROUGH: "fallthrough", | |
| FOR: "for", | |
| FUNC: "func", | |
| GO: "go", | |
| GOTO: "goto", | |
| IF: "if", | |
| IMPORT: "import", | |
| INTERFACE: "interface", | |
| MAP: "map", | |
| PACKAGE: "package", | |
| RANGE: "range", | |
| RETURN: "return", | |
| SELECT: "select", | |
| STRUCT: "struct", | |
| SWITCH: "switch", | |
| TYPE: "type", | |
| VAR: "var", | |
| } | |
| // String returns the string corresponding to the token tok. | |
| // For operators, delimiters, and keywords the string is the actual | |
| // token character sequence (e.g., for the token ADD, the string is | |
| // "+"). For all other tokens the string corresponds to the token | |
| // constant name (e.g. for the token IDENT, the string is "IDENT"). | |
| // | |
| func (tok Token) String() string { | |
| s := "" | |
| if 0 <= tok && tok < Token(len(tokens)) { | |
| s = tokens[tok] | |
| } | |
| if s == "" { | |
| s = "token(" + strconv.Itoa(int(tok)) + ")" | |
| } | |
| return s | |
| } | |
| // A set of constants for precedence-based expression parsing. | |
| // Non-operators have lowest precedence, followed by operators | |
| // starting with precedence 1 up to unary operators. The highest | |
| // precedence serves as "catch-all" precedence for selector, | |
| // indexing, and other operator and delimiter tokens. | |
| // | |
| const ( | |
| LowestPrec = 0 // non-operators | |
| UnaryPrec = 6 | |
| HighestPrec = 7 | |
| ) | |
| // Precedence returns the operator precedence of the binary | |
| // operator op. If op is not a binary operator, the result | |
| // is LowestPrecedence. | |
| // | |
| func (op Token) Precedence() int { | |
| switch op { | |
| case LOR: | |
| return 1 | |
| case LAND: | |
| return 2 | |
| case EQL, NEQ, LSS, LEQ, GTR, GEQ: | |
| return 3 | |
| case ADD, SUB, OR, XOR: | |
| return 4 | |
| case MUL, QUO, REM, SHL, SHR, AND, AND_NOT: | |
| return 5 | |
| } | |
| return LowestPrec | |
| } | |
| var keywords map[string]Token | |
| func init() { | |
| keywords = make(map[string]Token) | |
| for i := keyword_beg + 1; i < keyword_end; i++ { | |
| keywords[tokens[i]] = i | |
| } | |
| } | |
| // Lookup maps an identifier to its keyword token or IDENT (if not a keyword). | |
| // | |
| func Lookup(ident string) Token { | |
| if tok, is_keyword := keywords[ident]; is_keyword { | |
| return tok | |
| } | |
| return IDENT | |
| } | |
| // Predicates | |
| // IsLiteral returns true for tokens corresponding to identifiers | |
| // and basic type literals; it returns false otherwise. | |
| // | |
| func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_end } | |
| // IsOperator returns true for tokens corresponding to operators and | |
| // delimiters; it returns false otherwise. | |
| // | |
| func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end } | |
| // IsKeyword returns true for tokens corresponding to keywords; | |
| // it returns false otherwise. | |
| // | |
| func (tok Token) IsKeyword() bool { return keyword_beg < tok && tok < keyword_end } | |
| // IsExported reports whether name starts with an upper-case letter. | |
| // | |
| func IsExported(name string) bool { | |
| ch, _ := utf8.DecodeRuneInString(name) | |
| return unicode.IsUpper(ch) | |
| } | |
| // IsKeyword reports whether name is a Go keyword, such as "func" or "return". | |
| // | |
| func IsKeyword(name string) bool { | |
| // TODO: opt: use a perfect hash function instead of a global map. | |
| _, ok := keywords[name] | |
| return ok | |
| } | |
| // IsIdentifier reports whether name is a Go identifier, that is, a non-empty | |
| // string made up of letters, digits, and underscores, where the first character | |
| // is not a digit. Keywords are not identifiers. | |
| // | |
| func IsIdentifier(name string) bool { | |
| for i, c := range name { | |
| if !unicode.IsLetter(c) && c != '_' && (i == 0 || !unicode.IsDigit(c)) { | |
| return false | |
| } | |
| } | |
| return name != "" && !IsKeyword(name) | |
| } |