| // Copyright 2009 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| // This package defines constants representing the lexical |
| // tokens of the Go programming language and basic operations |
| // on tokens (printing, predicates). |
| // |
| package token |
| |
| import ( |
| "fmt"; |
| "strconv"; |
| ) |
| |
| |
| // Token is the set of lexical tokens of the Go programming language. |
| type Token int |
| |
| // The list of tokens. |
| const ( |
| // Special tokens |
| ILLEGAL Token = iota; |
| EOF; |
| COMMENT; |
| |
| literal_beg; |
| // Identifiers and basic type literals |
| // (these tokens stand for classes of literals) |
| IDENT; // main |
| INT; // 12345 |
| FLOAT; // 123.45 |
| CHAR; // 'a' |
| STRING; // "abc" |
| literal_end; |
| |
| operator_beg; |
| // Operators and delimiters |
| ADD; // + |
| SUB; // - |
| MUL; // * |
| QUO; // / |
| REM; // % |
| |
| AND; // & |
| OR; // | |
| XOR; // ^ |
| SHL; // << |
| SHR; // >> |
| AND_NOT; // &^ |
| |
| ADD_ASSIGN; // += |
| SUB_ASSIGN; // -= |
| MUL_ASSIGN; // *= |
| QUO_ASSIGN; // /= |
| REM_ASSIGN; // %= |
| |
| AND_ASSIGN; // &= |
| OR_ASSIGN; // |= |
| XOR_ASSIGN; // ^= |
| SHL_ASSIGN; // <<= |
| SHR_ASSIGN; // >>= |
| AND_NOT_ASSIGN; // &^= |
| |
| LAND; // && |
| LOR; // || |
| ARROW; // <- |
| INC; // ++ |
| DEC; // -- |
| |
| EQL; // == |
| LSS; // < |
| GTR; // > |
| ASSIGN; // = |
| NOT; // ! |
| |
| NEQ; // != |
| LEQ; // <= |
| GEQ; // >= |
| DEFINE; // := |
| ELLIPSIS; // ... |
| |
| LPAREN; // ( |
| LBRACK; // [ |
| LBRACE; // { |
| COMMA; // , |
| PERIOD; // . |
| |
| RPAREN; // ) |
| RBRACK; // ] |
| RBRACE; // } |
| SEMICOLON; // ; |
| COLON; // : |
| operator_end; |
| |
| keyword_beg; |
| // Keywords |
| BREAK; |
| CASE; |
| CHAN; |
| CONST; |
| CONTINUE; |
| |
| DEFAULT; |
| DEFER; |
| ELSE; |
| FALLTHROUGH; |
| FOR; |
| |
| FUNC; |
| GO; |
| GOTO; |
| IF; |
| IMPORT; |
| |
| INTERFACE; |
| MAP; |
| PACKAGE; |
| RANGE; |
| RETURN; |
| |
| SELECT; |
| STRUCT; |
| SWITCH; |
| TYPE; |
| VAR; |
| keyword_end; |
| ) |
| |
| |
| // At the moment we have no array literal syntax that lets us describe |
| // the index for each element - use a map for now to make sure they are |
| // in sync. |
| var tokens = map [Token] string { |
| ILLEGAL : "ILLEGAL", |
| |
| EOF : "EOF", |
| COMMENT : "COMMENT", |
| |
| IDENT : "IDENT", |
| INT : "INT", |
| FLOAT : "FLOAT", |
| CHAR : "CHAR", |
| STRING : "STRING", |
| |
| ADD : "+", |
| SUB : "-", |
| MUL : "*", |
| QUO : "/", |
| REM : "%", |
| |
| AND : "&", |
| OR : "|", |
| XOR : "^", |
| SHL : "<<", |
| SHR : ">>", |
| AND_NOT : "&^", |
| |
| ADD_ASSIGN : "+=", |
| SUB_ASSIGN : "-=", |
| MUL_ASSIGN : "*=", |
| QUO_ASSIGN : "/=", |
| REM_ASSIGN : "%=", |
| |
| AND_ASSIGN : "&=", |
| OR_ASSIGN : "|=", |
| XOR_ASSIGN : "^=", |
| SHL_ASSIGN : "<<=", |
| SHR_ASSIGN : ">>=", |
| AND_NOT_ASSIGN : "&^=", |
| |
| LAND : "&&", |
| LOR : "||", |
| ARROW : "<-", |
| INC : "++", |
| DEC : "--", |
| |
| EQL : "==", |
| LSS : "<", |
| GTR : ">", |
| ASSIGN : "=", |
| NOT : "!", |
| |
| NEQ : "!=", |
| LEQ : "<=", |
| GEQ : ">=", |
| DEFINE : ":=", |
| ELLIPSIS : "...", |
| |
| LPAREN : "(", |
| LBRACK : "[", |
| LBRACE : "{", |
| COMMA : ",", |
| PERIOD : ".", |
| |
| RPAREN : ")", |
| RBRACK : "]", |
| RBRACE : "}", |
| SEMICOLON : ";", |
| COLON : ":", |
| |
| BREAK : "break", |
| CASE : "case", |
| CHAN : "chan", |
| CONST : "const", |
| CONTINUE : "continue", |
| |
| DEFAULT : "default", |
| DEFER : "defer", |
| ELSE : "else", |
| FALLTHROUGH : "fallthrough", |
| FOR : "for", |
| |
| FUNC : "func", |
| GO : "go", |
| GOTO : "goto", |
| IF : "if", |
| IMPORT : "import", |
| |
| INTERFACE : "interface", |
| MAP : "map", |
| PACKAGE : "package", |
| RANGE : "range", |
| RETURN : "return", |
| |
| SELECT : "select", |
| STRUCT : "struct", |
| SWITCH : "switch", |
| TYPE : "type", |
| VAR : "var", |
| } |
| |
| |
| // String returns the string corresponding to the token tok. |
| // For operators, delimiters, and keywords the string is the actual |
| // token character sequence (e.g., for the token ADD, the string is |
| // "+"). For all other tokens the string corresponds to the token |
| // constant name (e.g. for the token IDENT, the string is "IDENT"). |
| // |
| func (tok Token) String() string { |
| if str, exists := tokens[tok]; exists { |
| return str; |
| } |
| return "token(" + strconv.Itoa(int(tok)) + ")"; |
| } |
| |
| |
| // A set of constants for precedence-based expression parsing. |
| // Non-operators have lowest precedence, followed by operators |
| // starting with precedence 1 up to unary operators. The highest |
| // precedence corresponds serves as "catch-all" precedence for |
| // selector, indexing, and other operator and delimiter tokens. |
| // |
| const ( |
| LowestPrec = 0; // non-operators |
| UnaryPrec = 7; |
| HighestPrec = 8; |
| ) |
| |
| |
| // Precedence returns the operator precedence of the binary |
| // operator op. If op is not a binary operator, the result |
| // is LowestPrecedence. |
| // |
| func (op Token) Precedence() int { |
| switch op { |
| case LOR: |
| return 1; |
| case LAND: |
| return 2; |
| case ARROW: |
| return 3; |
| case EQL, NEQ, LSS, LEQ, GTR, GEQ: |
| return 4; |
| case ADD, SUB, OR, XOR: |
| return 5; |
| case MUL, QUO, REM, SHL, SHR, AND, AND_NOT: |
| return 6; |
| } |
| return LowestPrec; |
| } |
| |
| |
| var keywords map [string] Token; |
| |
| func init() { |
| keywords = make(map [string] Token); |
| for i := keyword_beg + 1; i < keyword_end; i++ { |
| keywords[tokens[i]] = i; |
| } |
| } |
| |
| |
| // Lookup maps an identifier to its keyword token or IDENT (if not a keyword). |
| // |
| func Lookup(ident []byte) Token { |
| // TODO Maps with []byte key are illegal because []byte does not |
| // support == . Should find a more efficient solution eventually. |
| if tok, is_keyword := keywords[string(ident)]; is_keyword { |
| return tok; |
| } |
| return IDENT; |
| } |
| |
| |
| // Predicates |
| |
| // IsLiteral returns true for tokens corresponding to identifiers |
| // and basic type literals; returns false otherwise. |
| // |
| func (tok Token) IsLiteral() bool { |
| return literal_beg < tok && tok < literal_end; |
| } |
| |
| // IsOperator returns true for tokens corresponding to operators and |
| // delimiters; returns false otherwise. |
| // |
| func (tok Token) IsOperator() bool { |
| return operator_beg < tok && tok < operator_end; |
| } |
| |
| // IsKeyword returns true for tokens corresponding to keywords; |
| // returns false otherwise. |
| // |
| func (tok Token) IsKeyword() bool { |
| return keyword_beg < tok && tok < keyword_end; |
| } |
| |
| |
| // Token source positions are represented by a Position value. |
| // A Position is valid if the line number is > 0. |
| // |
| type Position struct { |
| Filename string; // filename, if any |
| Offset int; // byte offset, starting at 0 |
| Line int; // line number, starting at 1 |
| Column int; // column number, starting at 1 (character count) |
| } |
| |
| |
| // Pos is an accessor method for anonymous Position fields. |
| // It returns its receiver. |
| // |
| func (pos *Position) Pos() Position { |
| return *pos; |
| } |
| |
| |
| // IsValid returns true if the position is valid. |
| func (pos *Position) IsValid() bool { |
| return pos.Line > 0 |
| } |
| |
| |
| func (pos Position) String() string { |
| s := pos.Filename; |
| if pos.IsValid() { |
| if s != "" { |
| s += ":"; |
| } |
| s += fmt.Sprintf("%d:%d", pos.Line, pos.Column); |
| } |
| if s == "" { |
| s = "???"; |
| } |
| return s; |
| } |