| // lex.cc -- Go frontend lexer. |
| |
| // Copyright 2009 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| #include "go-system.h" |
| #include "go-diagnostics.h" |
| |
| #include "lex.h" |
| |
| // Manage mapping from keywords to the Keyword codes. |
| |
| class Keywords |
| { |
| public: |
| // The structure which maps keywords to codes. |
| struct Mapping |
| { |
| // Keyword string. |
| const char* keystring; |
| // Keyword code. |
| Keyword keycode; |
| }; |
| |
| // Return the parsecode corresponding to KEYSTRING, or |
| // KEYWORD_INVALID if it is not a keyword. |
| Keyword |
| keyword_to_code(const char* keyword, size_t len) const; |
| |
| // Return the string for a keyword. |
| const char* |
| keyword_to_string(Keyword) const; |
| |
| private: |
| static const Mapping mapping_[]; |
| static const int count_; |
| }; |
| |
| // Mapping from keyword string to keyword code. This array must be |
| // kept in sorted order, and the order must match the Keyword enum. |
| // Strings are looked up using bsearch. |
| |
| const Keywords::Mapping |
| Keywords::mapping_[] = |
| { |
| { NULL, KEYWORD_INVALID }, |
| { "__asm__", KEYWORD_ASM }, |
| { "break", KEYWORD_BREAK }, |
| { "case", KEYWORD_CASE }, |
| { "chan", KEYWORD_CHAN }, |
| { "const", KEYWORD_CONST }, |
| { "continue", KEYWORD_CONTINUE }, |
| { "default", KEYWORD_DEFAULT }, |
| { "defer", KEYWORD_DEFER }, |
| { "else", KEYWORD_ELSE }, |
| { "fallthrough", KEYWORD_FALLTHROUGH }, |
| { "for", KEYWORD_FOR }, |
| { "func", KEYWORD_FUNC }, |
| { "go", KEYWORD_GO }, |
| { "goto", KEYWORD_GOTO }, |
| { "if", KEYWORD_IF }, |
| { "import", KEYWORD_IMPORT }, |
| { "interface", KEYWORD_INTERFACE }, |
| { "map", KEYWORD_MAP }, |
| { "package", KEYWORD_PACKAGE }, |
| { "range", KEYWORD_RANGE }, |
| { "return", KEYWORD_RETURN }, |
| { "select", KEYWORD_SELECT }, |
| { "struct", KEYWORD_STRUCT }, |
| { "switch", KEYWORD_SWITCH }, |
| { "type", KEYWORD_TYPE }, |
| { "var", KEYWORD_VAR } |
| }; |
| |
| // Number of entries in the map. |
| |
| const int Keywords::count_ = |
| sizeof(Keywords::mapping_) / sizeof(Keywords::mapping_[0]); |
| |
| // Comparison function passed to bsearch. |
| |
| extern "C" |
| { |
| |
| struct Keywords_search_key |
| { |
| const char* str; |
| size_t len; |
| }; |
| |
| static int |
| keyword_compare(const void* keyv, const void* mapv) |
| { |
| const Keywords_search_key* key = |
| static_cast<const Keywords_search_key*>(keyv); |
| const Keywords::Mapping* map = |
| static_cast<const Keywords::Mapping*>(mapv); |
| if (map->keystring == NULL) |
| return 1; |
| int i = strncmp(key->str, map->keystring, key->len); |
| if (i != 0) |
| return i; |
| if (map->keystring[key->len] != '\0') |
| return -1; |
| return 0; |
| } |
| |
| } // End extern "C". |
| |
| // Convert a string to a keyword code. Return KEYWORD_INVALID if the |
| // string is not a keyword. |
| |
| Keyword |
| Keywords::keyword_to_code(const char* keyword, size_t len) const |
| { |
| Keywords_search_key key; |
| key.str = keyword; |
| key.len = len; |
| void* mapv = bsearch(&key, |
| this->mapping_, |
| this->count_, |
| sizeof(this->mapping_[0]), |
| keyword_compare); |
| if (mapv == NULL) |
| return KEYWORD_INVALID; |
| Mapping* map = static_cast<Mapping*>(mapv); |
| return map->keycode; |
| } |
| |
| // Convert a keyword code to a string. |
| |
| const char* |
| Keywords::keyword_to_string(Keyword code) const |
| { |
| go_assert(code > KEYWORD_INVALID && code < this->count_); |
| const Mapping* map = &this->mapping_[code]; |
| go_assert(map->keycode == code); |
| return map->keystring; |
| } |
| |
| // There is one instance of the Keywords class. |
| |
| static Keywords keywords; |
| |
| // Class Token. |
| |
| // Make a general token. |
| |
| Token::Token(Classification classification, Location location) |
| : classification_(classification), location_(location) |
| { |
| } |
| |
| // Destroy a token. |
| |
| Token::~Token() |
| { |
| this->clear(); |
| } |
| |
| // Clear a token--release memory. |
| |
| void |
| Token::clear() |
| { |
| if (this->classification_ == TOKEN_INTEGER |
| || this->classification_ == TOKEN_CHARACTER) |
| mpz_clear(this->u_.integer_value); |
| else if (this->classification_ == TOKEN_FLOAT |
| || this->classification_ == TOKEN_IMAGINARY) |
| mpfr_clear(this->u_.float_value); |
| } |
| |
| // Construct a token. |
| |
| Token::Token(const Token& tok) |
| : classification_(tok.classification_), location_(tok.location_) |
| { |
| switch (this->classification_) |
| { |
| case TOKEN_INVALID: |
| case TOKEN_EOF: |
| break; |
| case TOKEN_KEYWORD: |
| this->u_.keyword = tok.u_.keyword; |
| break; |
| case TOKEN_IDENTIFIER: |
| case TOKEN_STRING: |
| this->u_.string_value = tok.u_.string_value; |
| break; |
| case TOKEN_OPERATOR: |
| this->u_.op = tok.u_.op; |
| break; |
| case TOKEN_CHARACTER: |
| case TOKEN_INTEGER: |
| mpz_init_set(this->u_.integer_value, tok.u_.integer_value); |
| break; |
| case TOKEN_FLOAT: |
| case TOKEN_IMAGINARY: |
| mpfr_init_set(this->u_.float_value, tok.u_.float_value, MPFR_RNDN); |
| break; |
| default: |
| go_unreachable(); |
| } |
| } |
| |
| // Assign to a token. |
| |
| Token& |
| Token::operator=(const Token& tok) |
| { |
| this->clear(); |
| this->classification_ = tok.classification_; |
| this->location_ = tok.location_; |
| switch (tok.classification_) |
| { |
| case TOKEN_INVALID: |
| case TOKEN_EOF: |
| break; |
| case TOKEN_KEYWORD: |
| this->u_.keyword = tok.u_.keyword; |
| break; |
| case TOKEN_IDENTIFIER: |
| this->u_.identifier_value.name = tok.u_.identifier_value.name; |
| this->u_.identifier_value.is_exported = |
| tok.u_.identifier_value.is_exported; |
| break; |
| case TOKEN_STRING: |
| this->u_.string_value = tok.u_.string_value; |
| break; |
| case TOKEN_OPERATOR: |
| this->u_.op = tok.u_.op; |
| break; |
| case TOKEN_CHARACTER: |
| case TOKEN_INTEGER: |
| mpz_init_set(this->u_.integer_value, tok.u_.integer_value); |
| break; |
| case TOKEN_FLOAT: |
| case TOKEN_IMAGINARY: |
| mpfr_init_set(this->u_.float_value, tok.u_.float_value, MPFR_RNDN); |
| break; |
| default: |
| go_unreachable(); |
| } |
| return *this; |
| } |
| |
| // Print the token for debugging. |
| |
| void |
| Token::print(FILE* file) const |
| { |
| switch (this->classification_) |
| { |
| case TOKEN_INVALID: |
| fprintf(file, "invalid"); |
| break; |
| case TOKEN_EOF: |
| fprintf(file, "EOF"); |
| break; |
| case TOKEN_KEYWORD: |
| fprintf(file, "keyword %s", keywords.keyword_to_string(this->u_.keyword)); |
| break; |
| case TOKEN_IDENTIFIER: |
| fprintf(file, "identifier \"%s\"", this->u_.string_value->c_str()); |
| break; |
| case TOKEN_STRING: |
| fprintf(file, "quoted string \"%s\"", this->u_.string_value->c_str()); |
| break; |
| case TOKEN_CHARACTER: |
| fprintf(file, "character "); |
| mpz_out_str(file, 10, this->u_.integer_value); |
| break; |
| case TOKEN_INTEGER: |
| fprintf(file, "integer "); |
| mpz_out_str(file, 10, this->u_.integer_value); |
| break; |
| case TOKEN_FLOAT: |
| fprintf(file, "float "); |
| mpfr_out_str(file, 10, 0, this->u_.float_value, MPFR_RNDN); |
| break; |
| case TOKEN_IMAGINARY: |
| fprintf(file, "imaginary "); |
| mpfr_out_str(file, 10, 0, this->u_.float_value, MPFR_RNDN); |
| break; |
| case TOKEN_OPERATOR: |
| fprintf(file, "operator "); |
| switch (this->u_.op) |
| { |
| case OPERATOR_INVALID: |
| fprintf(file, "invalid"); |
| break; |
| case OPERATOR_OROR: |
| fprintf(file, "||"); |
| break; |
| case OPERATOR_ANDAND: |
| fprintf(file, "&&"); |
| break; |
| case OPERATOR_EQEQ: |
| fprintf(file, "=="); |
| break; |
| case OPERATOR_NOTEQ: |
| fprintf(file, "!="); |
| break; |
| case OPERATOR_LT: |
| fprintf(file, "<"); |
| break; |
| case OPERATOR_LE: |
| fprintf(file, "<="); |
| break; |
| case OPERATOR_GT: |
| fprintf(file, ">"); |
| break; |
| case OPERATOR_GE: |
| fprintf(file, ">="); |
| break; |
| case OPERATOR_PLUS: |
| fprintf(file, "+"); |
| break; |
| case OPERATOR_MINUS: |
| fprintf(file, "-"); |
| break; |
| case OPERATOR_OR: |
| fprintf(file, "|"); |
| break; |
| case OPERATOR_XOR: |
| fprintf(file, "^"); |
| break; |
| case OPERATOR_MULT: |
| fprintf(file, "*"); |
| break; |
| case OPERATOR_DIV: |
| fprintf(file, "/"); |
| break; |
| case OPERATOR_MOD: |
| fprintf(file, "%%"); |
| break; |
| case OPERATOR_LSHIFT: |
| fprintf(file, "<<"); |
| break; |
| case OPERATOR_RSHIFT: |
| fprintf(file, ">>"); |
| break; |
| case OPERATOR_AND: |
| fprintf(file, "&"); |
| break; |
| case OPERATOR_BITCLEAR: |
| fprintf(file, "&^"); |
| break; |
| case OPERATOR_NOT: |
| fprintf(file, "!"); |
| break; |
| case OPERATOR_CHANOP: |
| fprintf(file, "<-"); |
| break; |
| case OPERATOR_EQ: |
| fprintf(file, "="); |
| break; |
| case OPERATOR_PLUSEQ: |
| fprintf(file, "+="); |
| break; |
| case OPERATOR_MINUSEQ: |
| fprintf(file, "-="); |
| break; |
| case OPERATOR_OREQ: |
| fprintf(file, "|="); |
| break; |
| case OPERATOR_XOREQ: |
| fprintf(file, "^="); |
| break; |
| case OPERATOR_MULTEQ: |
| fprintf(file, "*="); |
| break; |
| case OPERATOR_DIVEQ: |
| fprintf(file, "/="); |
| break; |
| case OPERATOR_MODEQ: |
| fprintf(file, "%%="); |
| break; |
| case OPERATOR_LSHIFTEQ: |
| fprintf(file, "<<="); |
| break; |
| case OPERATOR_RSHIFTEQ: |
| fprintf(file, ">>="); |
| break; |
| case OPERATOR_ANDEQ: |
| fprintf(file, "&="); |
| break; |
| case OPERATOR_BITCLEAREQ: |
| fprintf(file, "&^="); |
| break; |
| case OPERATOR_PLUSPLUS: |
| fprintf(file, "++"); |
| break; |
| case OPERATOR_MINUSMINUS: |
| fprintf(file, "--"); |
| break; |
| case OPERATOR_COLON: |
| fprintf(file, ":"); |
| break; |
| case OPERATOR_COLONEQ: |
| fprintf(file, ":="); |
| break; |
| case OPERATOR_SEMICOLON: |
| fprintf(file, ";"); |
| break; |
| case OPERATOR_DOT: |
| fprintf(file, "."); |
| break; |
| case OPERATOR_COMMA: |
| fprintf(file, ","); |
| break; |
| case OPERATOR_LPAREN: |
| fprintf(file, "("); |
| break; |
| case OPERATOR_RPAREN: |
| fprintf(file, ")"); |
| break; |
| case OPERATOR_LCURLY: |
| fprintf(file, "{"); |
| break; |
| case OPERATOR_RCURLY: |
| fprintf(file, "}"); |
| break; |
| case OPERATOR_LSQUARE: |
| fprintf(file, "["); |
| break; |
| case OPERATOR_RSQUARE: |
| fprintf(file, "]"); |
| break; |
| default: |
| go_unreachable(); |
| } |
| break; |
| default: |
| go_unreachable(); |
| } |
| } |
| |
| // Class Lex. |
| |
| Lex::Lex(const char* input_file_name, FILE* input_file, Linemap* linemap) |
| : input_file_name_(input_file_name), input_file_(input_file), |
| linemap_(linemap), linebuf_(NULL), linebufsize_(120), linesize_(0), |
| lineoff_(0), lineno_(0), add_semi_at_eol_(false), pragmas_(0), |
| extern_(), linknames_(NULL) |
| { |
| this->linebuf_ = new char[this->linebufsize_]; |
| this->linemap_->start_file(input_file_name, 0); |
| } |
| |
| Lex::~Lex() |
| { |
| delete[] this->linebuf_; |
| } |
| |
| // Read a new line from the file. |
| |
| ssize_t |
| Lex::get_line() |
| { |
| char* buf = this->linebuf_; |
| size_t size = this->linebufsize_; |
| |
| FILE* file = this->input_file_; |
| size_t cur = 0; |
| while (true) |
| { |
| int c = getc(file); |
| if (c == EOF) |
| { |
| if (cur == 0) |
| return -1; |
| break; |
| } |
| if (cur + 1 >= size) |
| { |
| size_t ns = 2 * size + 1; |
| if (ns < size || static_cast<ssize_t>(ns) < 0) |
| go_error_at(this->location(), "out of memory"); |
| char* nb = new char[ns]; |
| memcpy(nb, buf, cur); |
| delete[] buf; |
| buf = nb; |
| size = ns; |
| } |
| buf[cur] = c; |
| ++cur; |
| |
| if (c == '\n') |
| break; |
| } |
| |
| buf[cur] = '\0'; |
| |
| this->linebuf_ = buf; |
| this->linebufsize_ = size; |
| |
| return cur; |
| } |
| |
| // See if we need to read a new line. Return true if there is a new |
| // line, false if we are at EOF. |
| |
| bool |
| Lex::require_line() |
| { |
| if (this->lineoff_ < this->linesize_) |
| return true; |
| |
| ssize_t got = this->get_line(); |
| if (got < 0) |
| return false; |
| ++this->lineno_; |
| this->linesize_= got; |
| this->lineoff_ = 0; |
| |
| this->linemap_->start_line(this->lineno_, this->linesize_); |
| |
| return true; |
| } |
| |
| // Get the current location. |
| |
| Location |
| Lex::location() const |
| { |
| return this->linemap_->get_location(this->lineoff_ + 1); |
| } |
| |
| // Get a location slightly before the current one. This is used for |
| // slightly more efficient handling of operator tokens. |
| |
| Location |
| Lex::earlier_location(int chars) const |
| { |
| return this->linemap_->get_location(this->lineoff_ + 1 - chars); |
| } |
| |
| // Get the next token. |
| |
| Token |
| Lex::next_token() |
| { |
| bool saw_cpp_comment = false; |
| while (true) |
| { |
| if (!this->require_line()) |
| { |
| bool add_semi_at_eol = this->add_semi_at_eol_; |
| this->add_semi_at_eol_ = false; |
| if (add_semi_at_eol) |
| return this->make_operator(OPERATOR_SEMICOLON, 1); |
| return this->make_eof_token(); |
| } |
| |
| if (!saw_cpp_comment) |
| this->extern_.clear(); |
| saw_cpp_comment = false; |
| |
| const char* p = this->linebuf_ + this->lineoff_; |
| const char* pend = this->linebuf_ + this->linesize_; |
| |
| while (p < pend) |
| { |
| unsigned char cc = *p; |
| switch (cc) |
| { |
| case ' ': case '\t': case '\r': |
| ++p; |
| // Skip whitespace quickly. |
| while (*p == ' ' || *p == '\t' || *p == '\r') |
| ++p; |
| break; |
| |
| case '\n': |
| { |
| ++p; |
| bool add_semi_at_eol = this->add_semi_at_eol_; |
| this->add_semi_at_eol_ = false; |
| if (add_semi_at_eol) |
| { |
| this->lineoff_ = p - this->linebuf_; |
| return this->make_operator(OPERATOR_SEMICOLON, 1); |
| } |
| } |
| break; |
| |
| case '/': |
| if (p[1] == '/') |
| { |
| this->lineoff_ = p + 2 - this->linebuf_; |
| this->skip_cpp_comment(); |
| p = pend; |
| if (p[-1] == '\n' && this->add_semi_at_eol_) |
| --p; |
| saw_cpp_comment = true; |
| } |
| else if (p[1] == '*') |
| { |
| this->lineoff_ = p + 2 - this->linebuf_; |
| Location location = this->location(); |
| bool found_newline = false; |
| if (!this->skip_c_comment(&found_newline)) |
| return Token::make_invalid_token(location); |
| if (found_newline && this->add_semi_at_eol_) |
| { |
| this->add_semi_at_eol_ = false; |
| return this->make_operator(OPERATOR_SEMICOLON, 1); |
| } |
| p = this->linebuf_ + this->lineoff_; |
| pend = this->linebuf_ + this->linesize_; |
| } |
| else if (p[1] == '=') |
| { |
| this->add_semi_at_eol_ = false; |
| this->lineoff_ = p + 2 - this->linebuf_; |
| return this->make_operator(OPERATOR_DIVEQ, 2); |
| } |
| else |
| { |
| this->add_semi_at_eol_ = false; |
| this->lineoff_ = p + 1 - this->linebuf_; |
| return this->make_operator(OPERATOR_DIV, 1); |
| } |
| break; |
| |
| case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': |
| case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': |
| case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': |
| case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': |
| case 'Y': case 'Z': |
| case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': |
| case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': |
| case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': |
| case 's': case 't': case 'u': case 'v': case 'w': case 'x': |
| case 'y': case 'z': |
| case '_': |
| this->lineoff_ = p - this->linebuf_; |
| return this->gather_identifier(); |
| |
| case '0': case '1': case '2': case '3': case '4': |
| case '5': case '6': case '7': case '8': case '9': |
| this->add_semi_at_eol_ = true; |
| this->lineoff_ = p - this->linebuf_; |
| return this->gather_number(); |
| |
| case '\'': |
| this->add_semi_at_eol_ = true; |
| this->lineoff_ = p - this->linebuf_; |
| return this->gather_character(); |
| |
| case '"': |
| this->add_semi_at_eol_ = true; |
| this->lineoff_ = p - this->linebuf_; |
| return this->gather_string(); |
| |
| case '`': |
| this->add_semi_at_eol_ = true; |
| this->lineoff_ = p - this->linebuf_; |
| return this->gather_raw_string(); |
| |
| case '<': |
| case '>': |
| case '&': |
| if (p + 2 < pend) |
| { |
| this->add_semi_at_eol_ = false; |
| Operator op = this->three_character_operator(cc, p[1], p[2]); |
| if (op != OPERATOR_INVALID) |
| { |
| this->lineoff_ = p + 3 - this->linebuf_; |
| return this->make_operator(op, 3); |
| } |
| } |
| // Fall through. |
| case '|': |
| case '=': |
| case '!': |
| case '+': |
| case '-': |
| case '^': |
| case '*': |
| // '/' handled above. |
| case '%': |
| case ':': |
| case ';': |
| case ',': |
| case '(': case ')': |
| case '{': case '}': |
| case '[': case ']': |
| { |
| this->add_semi_at_eol_ = false; |
| Operator op = this->two_character_operator(cc, p[1]); |
| int chars; |
| if (op != OPERATOR_INVALID) |
| { |
| ++p; |
| chars = 2; |
| } |
| else |
| { |
| op = this->one_character_operator(cc); |
| chars = 1; |
| } |
| this->lineoff_ = p + 1 - this->linebuf_; |
| return this->make_operator(op, chars); |
| } |
| |
| case '.': |
| if (p[1] >= '0' && p[1] <= '9') |
| { |
| this->add_semi_at_eol_ = true; |
| this->lineoff_ = p - this->linebuf_; |
| return this->gather_number(); |
| } |
| if (p[1] == '.' && p[2] == '.') |
| { |
| this->add_semi_at_eol_ = false; |
| this->lineoff_ = p + 3 - this->linebuf_; |
| return this->make_operator(OPERATOR_ELLIPSIS, 3); |
| } |
| this->add_semi_at_eol_ = false; |
| this->lineoff_ = p + 1 - this->linebuf_; |
| return this->make_operator(OPERATOR_DOT, 1); |
| |
| default: |
| { |
| unsigned int ci; |
| bool issued_error; |
| this->lineoff_ = p - this->linebuf_; |
| const char *pnext = this->advance_one_utf8_char(p, &ci, |
| &issued_error); |
| |
| // Ignore byte order mark at start of file. |
| if (ci == 0xfeff) |
| { |
| p = pnext; |
| break; |
| } |
| |
| if (Lex::is_unicode_letter(ci)) |
| return this->gather_identifier(); |
| |
| if (!issued_error) |
| go_error_at(this->location(), |
| "invalid character 0x%x in input file", |
| ci); |
| |
| p = pend; |
| |
| break; |
| } |
| } |
| } |
| |
| this->lineoff_ = p - this->linebuf_; |
| } |
| } |
| |
| // Fetch one UTF-8 character from a string. Set *VALUE to the value. |
| // Return the number of bytes read from the string. Returns 0 if the |
| // string does not point to a valid UTF-8 character. |
| |
| int |
| Lex::fetch_char(const char* p, unsigned int* value) |
| { |
| unsigned char c = *p; |
| if (c <= 0x7f) |
| { |
| *value = c; |
| return 1; |
| } |
| else if ((c & 0xe0) == 0xc0 |
| && (p[1] & 0xc0) == 0x80) |
| { |
| *value = (((c & 0x1f) << 6) |
| + (p[1] & 0x3f)); |
| if (*value <= 0x7f) |
| { |
| *value = 0xfffd; |
| return 0; |
| } |
| return 2; |
| } |
| else if ((c & 0xf0) == 0xe0 |
| && (p[1] & 0xc0) == 0x80 |
| && (p[2] & 0xc0) == 0x80) |
| { |
| *value = (((c & 0xf) << 12) |
| + ((p[1] & 0x3f) << 6) |
| + (p[2] & 0x3f)); |
| if (*value <= 0x7ff) |
| { |
| *value = 0xfffd; |
| return 0; |
| } |
| return 3; |
| } |
| else if ((c & 0xf8) == 0xf0 |
| && (p[1] & 0xc0) == 0x80 |
| && (p[2] & 0xc0) == 0x80 |
| && (p[3] & 0xc0) == 0x80) |
| { |
| *value = (((c & 0x7) << 18) |
| + ((p[1] & 0x3f) << 12) |
| + ((p[2] & 0x3f) << 6) |
| + (p[3] & 0x3f)); |
| if (*value <= 0xffff) |
| { |
| *value = 0xfffd; |
| return 0; |
| } |
| return 4; |
| } |
| else |
| { |
| /* Invalid encoding. Return the Unicode replacement |
| character. */ |
| *value = 0xfffd; |
| return 0; |
| } |
| } |
| |
| // Advance one UTF-8 character. Return the pointer beyond the |
| // character. Set *VALUE to the value. Set *ISSUED_ERROR if an error |
| // was issued. |
| |
| const char* |
| Lex::advance_one_utf8_char(const char* p, unsigned int* value, |
| bool* issued_error) |
| { |
| *issued_error = false; |
| |
| if (*p == '\0') |
| { |
| go_error_at(this->location(), "invalid NUL byte"); |
| *issued_error = true; |
| *value = 0; |
| return p + 1; |
| } |
| |
| int adv = Lex::fetch_char(p, value); |
| if (adv == 0) |
| { |
| go_error_at(this->location(), "invalid UTF-8 encoding"); |
| *issued_error = true; |
| return p + 1; |
| } |
| |
| // Warn about byte order mark, except at start of file. |
| if (*value == 0xfeff && (this->lineno_ != 1 || this->lineoff_ != 0)) |
| { |
| go_error_at(this->location(), "Unicode (UTF-8) BOM in middle of file"); |
| *issued_error = true; |
| } |
| |
| return p + adv; |
| } |
| |
| // Pick up an identifier. |
| |
| Token |
| Lex::gather_identifier() |
| { |
| const char* pstart = this->linebuf_ + this->lineoff_; |
| const char* p = pstart; |
| const char* pend = this->linebuf_ + this->linesize_; |
| bool is_first = true; |
| bool is_exported = false; |
| bool has_non_ascii_char = false; |
| std::string buf; |
| while (p < pend) |
| { |
| unsigned char cc = *p; |
| if (cc <= 0x7f) |
| { |
| if ((cc < 'A' || cc > 'Z') |
| && (cc < 'a' || cc > 'z') |
| && cc != '_' |
| && (cc < '0' || cc > '9')) |
| { |
| // Check for an invalid character here, as we get better |
| // error behaviour if we swallow them as part of the |
| // identifier we are building. |
| if ((cc >= ' ' && cc < 0x7f) |
| || cc == '\t' |
| || cc == '\r' |
| || cc == '\n') |
| break; |
| |
| this->lineoff_ = p - this->linebuf_; |
| go_error_at(this->location(), |
| "invalid character 0x%x in identifier", |
| cc); |
| if (!has_non_ascii_char) |
| { |
| buf.assign(pstart, p - pstart); |
| has_non_ascii_char = true; |
| } |
| if (!Lex::is_invalid_identifier(buf)) |
| buf.append("$INVALID$"); |
| } |
| ++p; |
| if (is_first) |
| { |
| is_exported = cc >= 'A' && cc <= 'Z'; |
| is_first = false; |
| } |
| if (has_non_ascii_char) |
| buf.push_back(cc); |
| } |
| else |
| { |
| unsigned int ci; |
| bool issued_error; |
| this->lineoff_ = p - this->linebuf_; |
| const char* pnext = this->advance_one_utf8_char(p, &ci, |
| &issued_error); |
| bool is_invalid = false; |
| if (!Lex::is_unicode_letter(ci) && !Lex::is_unicode_digit(ci)) |
| { |
| // There is no valid place for a non-ASCII character |
| // other than an identifier, so we get better error |
| // handling behaviour if we swallow this character after |
| // giving an error. |
| if (!issued_error) |
| go_error_at(this->location(), |
| "invalid character 0x%x in identifier", |
| ci); |
| is_invalid = true; |
| } |
| if (is_first) |
| { |
| is_exported = Lex::is_unicode_uppercase(ci); |
| is_first = false; |
| } |
| if (!has_non_ascii_char) |
| { |
| buf.assign(pstart, p - pstart); |
| has_non_ascii_char = true; |
| } |
| if (is_invalid && !Lex::is_invalid_identifier(buf)) |
| buf.append("$INVALID$"); |
| buf.append(p, pnext - p); |
| p = pnext; |
| } |
| } |
| Location location = this->location(); |
| this->add_semi_at_eol_ = true; |
| this->lineoff_ = p - this->linebuf_; |
| if (has_non_ascii_char) |
| return Token::make_identifier_token(buf, is_exported, location); |
| else |
| { |
| Keyword code = keywords.keyword_to_code(pstart, p - pstart); |
| if (code == KEYWORD_INVALID) |
| return Token::make_identifier_token(std::string(pstart, p - pstart), |
| is_exported, location); |
| else |
| { |
| switch (code) |
| { |
| case KEYWORD_BREAK: |
| case KEYWORD_CONTINUE: |
| case KEYWORD_FALLTHROUGH: |
| case KEYWORD_RETURN: |
| break; |
| default: |
| this->add_semi_at_eol_ = false; |
| break; |
| } |
| return Token::make_keyword_token(code, location); |
| } |
| } |
| } |
| |
| // Return whether C is a hex digit. |
| |
| bool |
| Lex::is_hex_digit(char c) |
| { |
| return ((c >= '0' && c <= '9') |
| || (c >= 'A' && c <= 'F') |
| || (c >= 'a' && c <= 'f')); |
| } |
| |
| // Return whether C is a valid digit in BASE. |
| |
| bool |
| Lex::is_base_digit(int base, char c) |
| { |
| switch (base) |
| { |
| case 2: |
| return c == '0' || c == '1'; |
| case 8: |
| return c >= '0' && c <= '7'; |
| case 10: |
| return c >= '0' && c <= '9'; |
| case 16: |
| return Lex::is_hex_digit(c); |
| default: |
| go_unreachable(); |
| } |
| } |
| |
| // not a hex value |
| #define NHV 100 |
| |
| // for use by Lex::hex_val |
| static const unsigned char hex_value_lookup_table[256] = |
| { |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // NUL SOH STX ETX EOT ENQ ACK BEL |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // BS HT LF VT FF CR SO SI |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // DLE DC1 DC2 DC3 DC4 NAK SYN ETB |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // CAN EM SUB ESC FS GS RS US |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // SP ! " # $ % & ' |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // ( ) * + , - . / |
| 0, 1, 2, 3, 4, 5, 6, 7, // 0 1 2 3 4 5 6 7 |
| 8, 9, NHV, NHV, NHV, NHV, NHV, NHV, // 8 9 : ; < = > ? |
| NHV, 10, 11, 12, 13, 14, 15, NHV, // @ A B C D E F G |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // H I J K L M N O |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // P Q R S T U V W |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // X Y Z [ \ ] ^ _ |
| NHV, 10, 11, 12, 13, 14, 15, NHV, // ` a b c d e f g |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // h i j k l m n o |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // p q r s t u v w |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // x y z { | } ~ |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV, // |
| NHV, NHV, NHV, NHV, NHV, NHV, NHV, NHV // |
| }; |
| |
| unsigned |
| Lex::hex_val(char c) |
| { |
| return hex_value_lookup_table[static_cast<unsigned char>(c)]; |
| } |
| |
| // Return whether an exponent could start at P, in base BASE. |
| |
| bool |
| Lex::could_be_exponent(int base, const char* p, const char* pend) |
| { |
| switch (base) |
| { |
| case 10: |
| if (*p != 'e' && *p != 'E') |
| return false; |
| break; |
| case 16: |
| if (*p != 'p' && *p != 'P') |
| return false; |
| break; |
| default: |
| go_unreachable(); |
| } |
| ++p; |
| if (p >= pend) |
| return false; |
| if (*p == '+' || *p == '-') |
| { |
| ++p; |
| if (p >= pend) |
| return false; |
| } |
| return *p >= '0' && *p <= '9'; |
| } |
| |
| // Pick up a number. |
| |
| Token |
| Lex::gather_number() |
| { |
| const char* pstart = this->linebuf_ + this->lineoff_; |
| const char* p = pstart; |
| const char* pend = this->linebuf_ + this->linesize_; |
| |
| Location location = this->location(); |
| |
| int base = 10; |
| std::string num; |
| if (*p == '0') |
| { |
| int basecheck; |
| int off; |
| if (p[1] == 'x' || p[1] == 'X') |
| { |
| base = 16; |
| basecheck = 16; |
| off = 2; |
| } |
| else if (p[1] == 'o' || p[1] == 'O') |
| { |
| base = 8; |
| basecheck = 8; |
| off = 2; |
| } |
| else if (p[1] == 'b' || p[1] == 'B') |
| { |
| base = 2; |
| basecheck = 2; |
| off = 2; |
| } |
| else |
| { |
| // Old style octal literal. May also be the start of a |
| // floating-point number (e.g., 09.2, 09e2) or an imaginary |
| // literal (e.g., 09i), so we have to accept decimal digits. |
| base = 8; |
| basecheck = 10; |
| off = 0; |
| } |
| |
| p += off; |
| if (*p == '_' && Lex::is_base_digit(basecheck, p[1])) |
| ++p; |
| |
| while (Lex::is_base_digit(basecheck, *p)) |
| { |
| num.push_back(*p); |
| ++p; |
| if (*p == '_' && Lex::is_base_digit(basecheck, p[1])) |
| ++p; |
| } |
| |
| // We must see at least one valid digit, except for a case like |
| // 0x.0p1. |
| if (num.length() == 0 && (base != 16 || *p != '.')) |
| { |
| go_error_at(this->location(), "invalid numeric literal"); |
| this->lineoff_ = p - this->linebuf_; |
| mpz_t val; |
| mpz_init_set_ui(val, 0); |
| Token ret = Token::make_integer_token(val, location); |
| mpz_clear(val); |
| return ret; |
| } |
| |
| bool is_float = false; |
| // A number that looks like an old-style octal literal might |
| // actually be the beginning of a floating-point or imaginary |
| // literal, in which case the value is decimal digits. Handle |
| // that case below by treating the leading '0' as decimal. |
| if (off == 0 |
| && (*p == '.' || *p == 'i' || Lex::could_be_exponent(10, p, pend))) |
| { |
| is_float = true; |
| base = 10; |
| } |
| else if (base == 16 |
| && (*p == '.' || Lex::could_be_exponent(16, p, pend))) |
| is_float = true; |
| |
| if (!is_float) |
| { |
| mpz_t val; |
| int r = mpz_init_set_str(val, num.c_str(), base); |
| if (r != 0) |
| { |
| const char *errword; |
| switch (base) |
| { |
| case 2: |
| errword = "binary"; |
| break; |
| case 8: |
| errword = "octal"; |
| break; |
| case 16: |
| errword = "hex"; |
| break; |
| default: |
| go_unreachable(); |
| } |
| go_error_at(this->location(), "invalid %s literal", errword); |
| } |
| |
| bool is_imaginary = *p == 'i'; |
| if (is_imaginary) |
| ++p; |
| |
| this->lineoff_ = p - this->linebuf_; |
| |
| if (*p == 'e' || *p == 'E' || *p == 'p' || *p == 'P') |
| { |
| go_error_at(location, |
| "invalid prefix for floating constant"); |
| this->skip_exponent(); |
| } |
| |
| if (!is_imaginary) |
| { |
| Token ret = Token::make_integer_token(val, location); |
| mpz_clear(val); |
| return ret; |
| } |
| else |
| { |
| mpfr_t ival; |
| mpfr_init_set_z(ival, val, MPFR_RNDN); |
| mpz_clear(val); |
| Token ret = Token::make_imaginary_token(ival, location); |
| mpfr_clear(ival); |
| return ret; |
| } |
| } |
| } |
| |
| while (p < pend) |
| { |
| if (*p == '_' && p[1] >= '0' && p[1] <= '9') |
| ++p; |
| else if (*p < '0' || *p > '9') |
| break; |
| num.push_back(*p); |
| ++p; |
| } |
| |
| if (*p != '.' && *p != 'i' && !Lex::could_be_exponent(base, p, pend)) |
| { |
| mpz_t val; |
| int r = mpz_init_set_str(val, num.c_str(), 10); |
| go_assert(r == 0); |
| |
| this->lineoff_ = p - this->linebuf_; |
| |
| if (*p == 'e' || *p == 'E' || *p == 'p' || *p == 'P') |
| { |
| go_error_at(location, |
| "invalid prefix for floating constant"); |
| this->skip_exponent(); |
| } |
| |
| Token ret = Token::make_integer_token(val, location); |
| mpz_clear(val); |
| return ret; |
| } |
| |
| if (*p != 'i') |
| { |
| bool dot = *p == '.'; |
| |
| num.push_back(*p); |
| ++p; |
| |
| if (!dot) |
| { |
| if (*p == '+' || *p == '-') |
| { |
| num.push_back(*p); |
| ++p; |
| } |
| } |
| |
| bool first = true; |
| while (p < pend) |
| { |
| if (!first && *p == '_' && Lex::is_base_digit(base, p[1])) |
| ++p; |
| else if (!Lex::is_base_digit(base, *p)) |
| break; |
| num.push_back(*p); |
| ++p; |
| first = false; |
| } |
| |
| if (dot && Lex::could_be_exponent(base, p, pend)) |
| { |
| num.push_back(*p); |
| ++p; |
| if (*p == '+' || *p == '-') |
| { |
| num.push_back(*p); |
| ++p; |
| } |
| first = true; |
| while (p < pend) |
| { |
| if (!first && *p == '_' && p[1] >= '0' && p[1] <= '9') |
| ++p; |
| else if (*p < '0' || *p > '9') |
| break; |
| num.push_back(*p); |
| ++p; |
| first = false; |
| } |
| } |
| else if (dot && base == 16) |
| { |
| go_error_at(this->location(), |
| "invalid hex floating-point literal with no exponent"); |
| num.append("p0"); |
| } |
| } |
| |
| mpfr_t val; |
| int r = mpfr_init_set_str(val, num.c_str(), base, MPFR_RNDN); |
| go_assert(r == 0); |
| |
| bool is_imaginary = *p == 'i'; |
| if (is_imaginary) |
| ++p; |
| |
| this->lineoff_ = p - this->linebuf_; |
| |
| if (*p == 'e' || *p == 'E' || *p == 'p' || *p == 'P') |
| { |
| go_error_at(location, |
| "invalid prefix for floating constant"); |
| this->skip_exponent(); |
| } |
| |
| if (is_imaginary) |
| { |
| Token ret = Token::make_imaginary_token(val, location); |
| mpfr_clear(val); |
| return ret; |
| } |
| else |
| { |
| Token ret = Token::make_float_token(val, location); |
| mpfr_clear(val); |
| return ret; |
| } |
| } |
| |
| // Skip an exponent after reporting an error. |
| |
| void |
| Lex::skip_exponent() |
| { |
| const char* p = this->linebuf_ + this->lineoff_; |
| const char* pend = this->linebuf_ + this->linesize_; |
| if (*p != 'e' && *p != 'E' && *p != 'p' && *p != 'P') |
| return; |
| ++p; |
| if (*p == '+' || *p == '-') |
| ++p; |
| while (p < pend) |
| { |
| if ((*p < '0' || *p > '9') && *p != '_') |
| break; |
| ++p; |
| } |
| this->lineoff_ = p - this->linebuf_; |
| } |
| |
| // Advance one character, possibly escaped. Return the pointer beyond |
| // the character. Set *VALUE to the character. Set *IS_CHARACTER if |
| // this is a character (e.g., 'a' or '\u1234') rather than a byte |
| // value (e.g., '\001'). |
| |
| const char* |
| Lex::advance_one_char(const char* p, bool is_single_quote, unsigned int* value, |
| bool* is_character) |
| { |
| *value = 0; |
| *is_character = true; |
| if (*p != '\\') |
| { |
| bool issued_error; |
| const char* ret = this->advance_one_utf8_char(p, value, &issued_error); |
| if (is_single_quote |
| && (*value == '\'' || *value == '\n') |
| && !issued_error) |
| go_error_at(this->location(), "invalid character literal"); |
| return ret; |
| } |
| else |
| { |
| ++p; |
| switch (*p) |
| { |
| case '0': case '1': case '2': case '3': |
| case '4': case '5': case '6': case '7': |
| *is_character = false; |
| if (p[1] >= '0' && p[1] <= '7' |
| && p[2] >= '0' && p[2] <= '7') |
| { |
| *value = ((Lex::octal_value(p[0]) << 6) |
| + (Lex::octal_value(p[1]) << 3) |
| + Lex::octal_value(p[2])); |
| if (*value > 255) |
| { |
| go_error_at(this->location(), "invalid octal constant"); |
| *value = 255; |
| } |
| return p + 3; |
| } |
| go_error_at(this->location(), "invalid octal character"); |
| return (p[1] >= '0' && p[1] <= '7' |
| ? p + 2 |
| : p + 1); |
| |
| case 'x': |
| *is_character = false; |
| if (Lex::is_hex_digit(p[1]) && Lex::is_hex_digit(p[2])) |
| { |
| *value = (Lex::hex_val(p[1]) << 4) + Lex::hex_val(p[2]); |
| return p + 3; |
| } |
| go_error_at(this->location(), "invalid hex character"); |
| return (Lex::is_hex_digit(p[1]) |
| ? p + 2 |
| : p + 1); |
| |
| case 'a': |
| *value = '\a'; |
| return p + 1; |
| case 'b': |
| *value = '\b'; |
| return p + 1; |
| case 'f': |
| *value = '\f'; |
| return p + 1; |
| case 'n': |
| *value = '\n'; |
| return p + 1; |
| case 'r': |
| *value = '\r'; |
| return p + 1; |
| case 't': |
| *value = '\t'; |
| return p + 1; |
| case 'v': |
| *value = '\v'; |
| return p + 1; |
| case '\\': |
| *value = '\\'; |
| return p + 1; |
| case '\'': |
| if (!is_single_quote) |
| go_error_at(this->location(), "invalid quoted character"); |
| *value = '\''; |
| return p + 1; |
| case '"': |
| if (is_single_quote) |
| go_error_at(this->location(), "invalid quoted character"); |
| *value = '"'; |
| return p + 1; |
| |
| case 'u': |
| if (Lex::is_hex_digit(p[1]) && Lex::is_hex_digit(p[2]) |
| && Lex::is_hex_digit(p[3]) && Lex::is_hex_digit(p[4])) |
| { |
| *value = ((Lex::hex_val(p[1]) << 12) |
| + (Lex::hex_val(p[2]) << 8) |
| + (Lex::hex_val(p[3]) << 4) |
| + Lex::hex_val(p[4])); |
| if (*value >= 0xd800 && *value < 0xe000) |
| { |
| go_error_at(this->location(), |
| "invalid unicode code point 0x%x", |
| *value); |
| // Use the replacement character. |
| *value = 0xfffd; |
| } |
| return p + 5; |
| } |
| go_error_at(this->location(), "invalid little unicode code point"); |
| return p + 1; |
| |
| case 'U': |
| if (Lex::is_hex_digit(p[1]) && Lex::is_hex_digit(p[2]) |
| && Lex::is_hex_digit(p[3]) && Lex::is_hex_digit(p[4]) |
| && Lex::is_hex_digit(p[5]) && Lex::is_hex_digit(p[6]) |
| && Lex::is_hex_digit(p[7]) && Lex::is_hex_digit(p[8])) |
| { |
| *value = ((Lex::hex_val(p[1]) << 28) |
| + (Lex::hex_val(p[2]) << 24) |
| + (Lex::hex_val(p[3]) << 20) |
| + (Lex::hex_val(p[4]) << 16) |
| + (Lex::hex_val(p[5]) << 12) |
| + (Lex::hex_val(p[6]) << 8) |
| + (Lex::hex_val(p[7]) << 4) |
| + Lex::hex_val(p[8])); |
| if (*value > 0x10ffff |
| || (*value >= 0xd800 && *value < 0xe000)) |
| { |
| go_error_at(this->location(), |
| "invalid unicode code point 0x%x", |
| *value); |
| // Use the replacement character. |
| *value = 0xfffd; |
| } |
| return p + 9; |
| } |
| go_error_at(this->location(), "invalid big unicode code point"); |
| return p + 1; |
| |
| default: |
| go_error_at(this->location(), "invalid character after %<\\%>"); |
| *value = *p; |
| return p + 1; |
| } |
| } |
| } |
| |
| // Append V to STR. IS_CHARACTER is true for a character which should |
| // be stored in UTF-8, false for a general byte value which should be |
| // stored directly. |
| |
| void |
| Lex::append_char(unsigned int v, bool is_character, std::string* str, |
| Location location) |
| { |
| char buf[4]; |
| size_t len; |
| if (v <= 0x7f || !is_character) |
| { |
| buf[0] = v; |
| len = 1; |
| } |
| else if (v <= 0x7ff) |
| { |
| buf[0] = 0xc0 + (v >> 6); |
| buf[1] = 0x80 + (v & 0x3f); |
| len = 2; |
| } |
| else |
| { |
| if (v > 0x10ffff) |
| { |
| go_warning_at(location, 0, |
| "unicode code point 0x%x out of range in string", v); |
| // Turn it into the "replacement character". |
| v = 0xfffd; |
| } |
| if (v >= 0xd800 && v < 0xe000) |
| { |
| go_warning_at(location, 0, |
| "unicode code point 0x%x is invalid surrogate pair", v); |
| v = 0xfffd; |
| } |
| if (v <= 0xffff) |
| { |
| buf[0] = 0xe0 + (v >> 12); |
| buf[1] = 0x80 + ((v >> 6) & 0x3f); |
| buf[2] = 0x80 + (v & 0x3f); |
| len = 3; |
| } |
| else |
| { |
| buf[0] = 0xf0 + (v >> 18); |
| buf[1] = 0x80 + ((v >> 12) & 0x3f); |
| buf[2] = 0x80 + ((v >> 6) & 0x3f); |
| buf[3] = 0x80 + (v & 0x3f); |
| len = 4; |
| } |
| } |
| str->append(buf, len); |
| } |
| |
| // Pick up a character literal. |
| |
| Token |
| Lex::gather_character() |
| { |
| ++this->lineoff_; |
| const char* pstart = this->linebuf_ + this->lineoff_; |
| const char* p = pstart; |
| |
| unsigned int value; |
| bool is_character; |
| p = this->advance_one_char(p, true, &value, &is_character); |
| |
| if (*p != '\'') |
| { |
| go_error_at(this->location(), "unterminated character constant"); |
| this->lineoff_ = p - this->linebuf_; |
| return this->make_invalid_token(); |
| } |
| |
| mpz_t val; |
| mpz_init_set_ui(val, value); |
| |
| Location location = this->location(); |
| this->lineoff_ = p + 1 - this->linebuf_; |
| Token ret = Token::make_character_token(val, location); |
| mpz_clear(val); |
| return ret; |
| } |
| |
| // Pick up a quoted string. |
| |
| Token |
| Lex::gather_string() |
| { |
| const char* pstart = this->linebuf_ + this->lineoff_ + 1; |
| const char* p = pstart; |
| const char* pend = this->linebuf_ + this->linesize_; |
| |
| std::string value; |
| while (*p != '"') |
| { |
| Location loc = this->location(); |
| unsigned int c; |
| bool is_character; |
| this->lineoff_ = p - this->linebuf_; |
| p = this->advance_one_char(p, false, &c, &is_character); |
| if (p >= pend) |
| { |
| go_error_at(this->location(), "unterminated string"); |
| --p; |
| break; |
| } |
| Lex::append_char(c, is_character, &value, loc); |
| } |
| |
| Location location = this->location(); |
| this->lineoff_ = p + 1 - this->linebuf_; |
| return Token::make_string_token(value, location); |
| } |
| |
| // Pick up a raw string. |
| |
| Token |
| Lex::gather_raw_string() |
| { |
| const char* p = this->linebuf_ + this->lineoff_ + 1; |
| const char* pend = this->linebuf_ + this->linesize_; |
| Location location = this->location(); |
| |
| std::string value; |
| while (true) |
| { |
| while (p < pend) |
| { |
| if (*p == '`') |
| { |
| this->lineoff_ = p + 1 - this->linebuf_; |
| return Token::make_string_token(value, location); |
| } |
| Location loc = this->location(); |
| unsigned int c; |
| bool issued_error; |
| this->lineoff_ = p - this->linebuf_; |
| p = this->advance_one_utf8_char(p, &c, &issued_error); |
| // "Carriage return characters ('\r') inside raw string literals |
| // are discarded from the raw string value." |
| if (c != '\r') |
| Lex::append_char(c, true, &value, loc); |
| } |
| this->lineoff_ = p - this->linebuf_; |
| if (!this->require_line()) |
| { |
| go_error_at(location, "unterminated raw string"); |
| return Token::make_string_token(value, location); |
| } |
| p = this->linebuf_ + this->lineoff_; |
| pend = this->linebuf_ + this->linesize_; |
| } |
| } |
| |
| // If C1 C2 C3 are a three character operator, return the code. |
| |
| Operator |
| Lex::three_character_operator(char c1, char c2, char c3) |
| { |
| if (c3 == '=') |
| { |
| if (c1 == '<' && c2 == '<') |
| return OPERATOR_LSHIFTEQ; |
| else if (c1 == '>' && c2 == '>') |
| return OPERATOR_RSHIFTEQ; |
| else if (c1 == '&' && c2 == '^') |
| return OPERATOR_BITCLEAREQ; |
| } |
| return OPERATOR_INVALID; |
| } |
| |
| // If C1 C2 are a two character operator, return the code. |
| |
| Operator |
| Lex::two_character_operator(char c1, char c2) |
| { |
| switch (c1) |
| { |
| case '|': |
| if (c2 == '|') |
| return OPERATOR_OROR; |
| else if (c2 == '=') |
| return OPERATOR_OREQ; |
| break; |
| case '&': |
| if (c2 == '&') |
| return OPERATOR_ANDAND; |
| else if (c2 == '^') |
| return OPERATOR_BITCLEAR; |
| else if (c2 == '=') |
| return OPERATOR_ANDEQ; |
| break; |
| case '^': |
| if (c2 == '=') |
| return OPERATOR_XOREQ; |
| break; |
| case '=': |
| if (c2 == '=') |
| return OPERATOR_EQEQ; |
| break; |
| case '!': |
| if (c2 == '=') |
| return OPERATOR_NOTEQ; |
| break; |
| case '<': |
| if (c2 == '=') |
| return OPERATOR_LE; |
| else if (c2 == '<') |
| return OPERATOR_LSHIFT; |
| else if (c2 == '-') |
| return OPERATOR_CHANOP; |
| break; |
| case '>': |
| if (c2 == '=') |
| return OPERATOR_GE; |
| else if (c2 == '>') |
| return OPERATOR_RSHIFT; |
| break; |
| case '*': |
| if (c2 == '=') |
| return OPERATOR_MULTEQ; |
| break; |
| case '/': |
| if (c2 == '=') |
| return OPERATOR_DIVEQ; |
| break; |
| case '%': |
| if (c2 == '=') |
| return OPERATOR_MODEQ; |
| break; |
| case '+': |
| if (c2 == '+') |
| { |
| this->add_semi_at_eol_ = true; |
| return OPERATOR_PLUSPLUS; |
| } |
| else if (c2 == '=') |
| return OPERATOR_PLUSEQ; |
| break; |
| case '-': |
| if (c2 == '-') |
| { |
| this->add_semi_at_eol_ = true; |
| return OPERATOR_MINUSMINUS; |
| } |
| else if (c2 == '=') |
| return OPERATOR_MINUSEQ; |
| break; |
| case ':': |
| if (c2 == '=') |
| return OPERATOR_COLONEQ; |
| break; |
| default: |
| break; |
| } |
| return OPERATOR_INVALID; |
| } |
| |
| // If character C is an operator, return the code. |
| |
| Operator |
| Lex::one_character_operator(char c) |
| { |
| switch (c) |
| { |
| case '<': |
| return OPERATOR_LT; |
| case '>': |
| return OPERATOR_GT; |
| case '+': |
| return OPERATOR_PLUS; |
| case '-': |
| return OPERATOR_MINUS; |
| case '|': |
| return OPERATOR_OR; |
| case '^': |
| return OPERATOR_XOR; |
| case '*': |
| return OPERATOR_MULT; |
| case '/': |
| return OPERATOR_DIV; |
| case '%': |
| return OPERATOR_MOD; |
| case '&': |
| return OPERATOR_AND; |
| case '!': |
| return OPERATOR_NOT; |
| case '=': |
| return OPERATOR_EQ; |
| case ':': |
| return OPERATOR_COLON; |
| case ';': |
| return OPERATOR_SEMICOLON; |
| case '.': |
| return OPERATOR_DOT; |
| case ',': |
| return OPERATOR_COMMA; |
| case '(': |
| return OPERATOR_LPAREN; |
| case ')': |
| this->add_semi_at_eol_ = true; |
| return OPERATOR_RPAREN; |
| case '{': |
| return OPERATOR_LCURLY; |
| case '}': |
| this->add_semi_at_eol_ = true; |
| return OPERATOR_RCURLY; |
| case '[': |
| return OPERATOR_LSQUARE; |
| case ']': |
| this->add_semi_at_eol_ = true; |
| return OPERATOR_RSQUARE; |
| default: |
| return OPERATOR_INVALID; |
| } |
| } |
| |
| // Skip a C-style comment. |
| |
| bool |
| Lex::skip_c_comment(bool* found_newline) |
| { |
| while (true) |
| { |
| if (!this->require_line()) |
| { |
| go_error_at(this->location(), "unterminated comment"); |
| return false; |
| } |
| |
| const char* p = this->linebuf_ + this->lineoff_; |
| const char* pend = this->linebuf_ + this->linesize_; |
| |
| while (p < pend) |
| { |
| if (p[0] == '*' && p + 1 < pend && p[1] == '/') |
| { |
| this->lineoff_ = p + 2 - this->linebuf_; |
| return true; |
| } |
| |
| if (p[0] == '\n') |
| *found_newline = true; |
| |
| this->lineoff_ = p - this->linebuf_; |
| unsigned int c; |
| bool issued_error; |
| p = this->advance_one_utf8_char(p, &c, &issued_error); |
| } |
| |
| this->lineoff_ = p - this->linebuf_; |
| } |
| } |
| |
| // Skip a C++-style comment. |
| |
| void |
| Lex::skip_cpp_comment() |
| { |
| // Ensure that if EXTERN_ is set, it means that we just saw a |
| // //extern comment. |
| this->extern_.clear(); |
| |
| Location loc = this->location(); |
| size_t lineoff = this->lineoff_; |
| |
| const char* p = this->linebuf_ + lineoff; |
| const char* pend = this->linebuf_ + this->linesize_; |
| |
| const char* pcheck = p; |
| bool saw_error = false; |
| while (pcheck < pend) |
| { |
| this->lineoff_ = pcheck - this->linebuf_; |
| unsigned int c; |
| bool issued_error; |
| pcheck = this->advance_one_utf8_char(pcheck, &c, &issued_error); |
| if (issued_error) |
| saw_error = true; |
| } |
| |
| if (saw_error) |
| return; |
| |
| // Recognize various magic comments at the start of a line. |
| |
| if (lineoff != 2) |
| { |
| // Not at the start of the line. (lineoff == 2 because of the |
| // two characters in "//"). |
| return; |
| } |
| |
| while (pend > p |
| && (pend[-1] == ' ' || pend[-1] == '\t' |
| || pend[-1] == '\r' || pend[-1] == '\n')) |
| --pend; |
| |
| // A C++ comment at the start of the line of the form |
| // //line FILE:LINENO |
| // is interpreted as setting the file name and line number of the |
| // next source line. |
| if (pend - p > 5 && memcmp(p, "line ", 5) == 0) |
| { |
| p += 5; |
| while (p < pend && *p == ' ') |
| ++p; |
| const char* pcolon = static_cast<const char*>(memchr(p, ':', pend - p)); |
| if (pcolon != NULL |
| && pcolon[1] >= '0' |
| && pcolon[1] <= '9') |
| { |
| char* plend; |
| long lineno = strtol(pcolon + 1, &plend, 10); |
| if (plend > pcolon + 1 |
| && (plend == pend |
| || *plend < '0' |
| || *plend > '9') |
| && lineno > 0 |
| && lineno < 0x7fffffff) |
| { |
| unsigned int filelen = pcolon - p; |
| char* file = new char[filelen + 1]; |
| memcpy(file, p, filelen); |
| file[filelen] = '\0'; |
| |
| this->linemap_->start_file(file, lineno); |
| this->lineno_ = lineno - 1; |
| |
| p = plend; |
| } |
| } |
| return; |
| } |
| |
| // As a special gccgo extension, a C++ comment at the start of the |
| // line of the form |
| // //extern NAME |
| // which immediately precedes a function declaration means that the |
| // external name of the function declaration is NAME. This is |
| // normally used to permit Go code to call a C function. |
| if (pend - p > 7 && memcmp(p, "extern ", 7) == 0) |
| { |
| p += 7; |
| while (p < pend && (*p == ' ' || *p == '\t')) |
| ++p; |
| if (pend > p) |
| this->extern_ = std::string(p, pend - p); |
| return; |
| } |
| |
| // All other special comments start with "go:". |
| |
| if (pend - p < 4 || memcmp(p, "go:", 3) != 0) |
| return; |
| |
| const char *ps = p + 3; |
| while (ps < pend && *ps != ' ' && *ps != '\t') |
| ++ps; |
| std::string verb = std::string(p, ps - p); |
| |
| if (verb == "go:linkname") |
| { |
| // As in the gc compiler, set the external link name for a Go symbol. |
| std::string go_name; |
| std::string ext_name; |
| bool is_exported = false; |
| if (ps < pend) |
| { |
| while (ps < pend && (*ps == ' ' || *ps == '\t')) |
| ++ps; |
| if (ps < pend) |
| { |
| const char* pg = ps; |
| |
| unsigned int c; |
| bool issued_error; |
| ps = this->advance_one_utf8_char(ps, &c, &issued_error); |
| is_exported = Lex::is_unicode_uppercase(c); |
| |
| while (ps < pend && *ps != ' ' && *ps != '\t') |
| ++ps; |
| if (ps <= pend) |
| go_name = std::string(pg, ps - pg); |
| while (ps < pend && (*ps == ' ' || *ps == '\t')) |
| ++ps; |
| } |
| if (ps < pend) |
| { |
| const char* pc = ps; |
| while (ps < pend && *ps != ' ' && *ps != '\t') |
| ++ps; |
| if (ps <= pend) |
| ext_name = std::string(pc, ps - pc); |
| } |
| if (ps != pend) |
| { |
| go_name.clear(); |
| ext_name.clear(); |
| } |
| } |
| if (go_name.empty()) |
| go_error_at(loc, "usage: %<//go:linkname%> localname [linkname]"); |
| else |
| { |
| if (this->linknames_ == NULL) |
| this->linknames_ = new Linknames(); |
| (*this->linknames_)[go_name] = Linkname(ext_name, is_exported, loc); |
| } |
| } |
| else if (verb == "go:nointerface") |
| { |
| // For field tracking analysis: a //go:nointerface comment means |
| // that the next interface method should not be stored in the |
| // type descriptor. This permits it to be discarded if it is |
| // not needed. |
| this->pragmas_ |= GOPRAGMA_NOINTERFACE; |
| } |
| else if (verb == "go:noescape") |
| { |
| // Applies to the next function declaration. Any arguments do |
| // not escape. |
| // FIXME: Not implemented. |
| this->pragmas_ |= GOPRAGMA_NOESCAPE; |
| } |
| else if (verb == "go:nosplit") |
| { |
| // Applies to the next function. Do not split the stack when |
| // entering the function. |
| this->pragmas_ |= GOPRAGMA_NOSPLIT; |
| } |
| else if (verb == "go:noinline") |
| { |
| // Applies to the next function. Do not inline the function. |
| this->pragmas_ |= GOPRAGMA_NOINLINE; |
| } |
| else if (verb == "go:notinheap") |
| { |
| // Applies to the next type. The type does not live in the heap. |
| this->pragmas_ |= GOPRAGMA_NOTINHEAP; |
| } |
| else if (verb == "go:systemstack") |
| { |
| // Applies to the next function. It must run on the system stack. |
| // FIXME: Should only work when compiling the runtime package. |
| // FIXME: Not implemented. |
| this->pragmas_ |= GOPRAGMA_SYSTEMSTACK; |
| } |
| else if (verb == "go:nowritebarrier") |
| { |
| // Applies to the next function. If the function needs to use |
| // any write barriers, it should emit an error instead. |
| // FIXME: Should only work when compiling the runtime package. |
| this->pragmas_ |= GOPRAGMA_NOWRITEBARRIER; |
| } |
| else if (verb == "go:nowritebarrierrec") |
| { |
| // Applies to the next function. If the function, or any |
| // function that it calls, needs to use any write barriers, it |
| // should emit an error instead. |
| // FIXME: Should only work when compiling the runtime package. |
| this->pragmas_ |= GOPRAGMA_NOWRITEBARRIERREC; |
| } |
| else if (verb == "go:yeswritebarrierrec") |
| { |
| // Applies to the next function. Disables go:nowritebarrierrec |
| // when looking at callees; write barriers are permitted here. |
| // FIXME: Should only work when compiling the runtime package. |
| this->pragmas_ |= GOPRAGMA_YESWRITEBARRIERREC; |
| } |
| else if (verb == "go:cgo_unsafe_args") |
| { |
| // Applies to the next function. Taking the address of any |
| // argument implies taking the address of all arguments. |
| // FIXME: Not implemented. |
| this->pragmas_ |= GOPRAGMA_CGOUNSAFEARGS; |
| } |
| else if (verb == "go:uintptrescapes") |
| { |
| // Applies to the next function. If an argument is a pointer |
| // converted to uintptr, then the pointer escapes. |
| // FIXME: Not implemented. |
| this->pragmas_ |= GOPRAGMA_UINTPTRESCAPES; |
| } |
| } |
| |
| // The Unicode tables use this struct. |
| |
| struct Unicode_range |
| { |
| // The low end of the range. |
| unsigned int low; |
| // The high end of the range. |
| unsigned int high; |
| // The stride. This entries represents low, low + stride, low + 2 * |
| // stride, etc., up to high. |
| unsigned int stride; |
| }; |
| |
| // A table of whitespace characters--Unicode code points classified as |
| // "Space", "C" locale whitespace characters, the "next line" control |
| // character (0085), the line separator (2028), the paragraph |
| // separator (2029), and the "zero-width non-break space" (feff). |
| |
| static const Unicode_range unicode_space[] = |
| { |
| { 0x0009, 0x000d, 1 }, |
| { 0x0020, 0x0020, 1 }, |
| { 0x0085, 0x0085, 1 }, |
| { 0x00a0, 0x00a0, 1 }, |
| { 0x1680, 0x1680, 1 }, |
| { 0x180e, 0x180e, 1 }, |
| { 0x2000, 0x200a, 1 }, |
| { 0x2028, 0x2029, 1 }, |
| { 0x202f, 0x202f, 1 }, |
| { 0x205f, 0x205f, 1 }, |
| { 0x3000, 0x3000, 1 }, |
| { 0xfeff, 0xfeff, 1 }, |
| }; |
| |
| // A table of Unicode digits--Unicode code points classified as |
| // "Digit". |
| |
| static const Unicode_range unicode_digits[] = |
| { |
| { 0x0030, 0x0039, 1}, |
| { 0x0660, 0x0669, 1}, |
| { 0x06f0, 0x06f9, 1}, |
| { 0x07c0, 0x07c9, 1}, |
| { 0x0966, 0x096f, 1}, |
| { 0x09e6, 0x09ef, 1}, |
| { 0x0a66, 0x0a6f, 1}, |
| { 0x0ae6, 0x0aef, 1}, |
| { 0x0b66, 0x0b6f, 1}, |
| { 0x0be6, 0x0bef, 1}, |
| { 0x0c66, 0x0c6f, 1}, |
| { 0x0ce6, 0x0cef, 1}, |
| { 0x0d66, 0x0d6f, 1}, |
| { 0x0e50, 0x0e59, 1}, |
| { 0x0ed0, 0x0ed9, 1}, |
| { 0x0f20, 0x0f29, 1}, |
| { 0x1040, 0x1049, 1}, |
| { 0x17e0, 0x17e9, 1}, |
| { 0x1810, 0x1819, 1}, |
| { 0x1946, 0x194f, 1}, |
| { 0x19d0, 0x19d9, 1}, |
| { 0x1b50, 0x1b59, 1}, |
| { 0xff10, 0xff19, 1}, |
| { 0x104a0, 0x104a9, 1}, |
| { 0x1d7ce, 0x1d7ff, 1}, |
| }; |
| |
| // A table of Unicode letters--Unicode code points classified as |
| // "Letter". |
| |
| static const Unicode_range unicode_letters[] = |
| { |
| { 0x0041, 0x005a, 1}, |
| { 0x0061, 0x007a, 1}, |
| { 0x00aa, 0x00b5, 11}, |
| { 0x00ba, 0x00c0, 6}, |
| { 0x00c1, 0x00d6, 1}, |
| { 0x00d8, 0x00f6, 1}, |
| { 0x00f8, 0x02c1, 1}, |
| { 0x02c6, 0x02d1, 1}, |
| { 0x02e0, 0x02e4, 1}, |
| { 0x02ec, 0x02ee, 2}, |
| { 0x0370, 0x0374, 1}, |
| { 0x0376, 0x0377, 1}, |
| { 0x037a, 0x037d, 1}, |
| { 0x037f, 0x0386, 7}, |
| { 0x0388, 0x038a, 1}, |
| { 0x038c, 0x038e, 2}, |
| { 0x038f, 0x03a1, 1}, |
| { 0x03a3, 0x03f5, 1}, |
| { 0x03f7, 0x0481, 1}, |
| { 0x048a, 0x052f, 1}, |
| { 0x0531, 0x0556, 1}, |
| { 0x0559, 0x0561, 8}, |
| { 0x0562, 0x0587, 1}, |
| { 0x05d0, 0x05ea, 1}, |
| { 0x05f0, 0x05f2, 1}, |
| { 0x0620, 0x064a, 1}, |
| { 0x066e, 0x066f, 1}, |
| { 0x0671, 0x06d3, 1}, |
| { 0x06d5, 0x06e5, 16}, |
| { 0x06e6, 0x06ee, 8}, |
| { 0x06ef, 0x06fa, 11}, |
| { 0x06fb, 0x06fc, 1}, |
| { 0x06ff, 0x0710, 17}, |
| { 0x0712, 0x072f, 1}, |
| { 0x074d, 0x07a5, 1}, |
| { 0x07b1, 0x07ca, 25}, |
| { 0x07cb, 0x07ea, 1}, |
| { 0x07f4, 0x07f5, 1}, |
| { 0x07fa, 0x0800, 6}, |
| { 0x0801, 0x0815, 1}, |
| { 0x081a, 0x0824, 10}, |
| { 0x0828, 0x0840, 24}, |
| { 0x0841, 0x0858, 1}, |
| { 0x08a0, 0x08b4, 1}, |
| { 0x0904, 0x0939, 1}, |
| { 0x093d, 0x0950, 19}, |
| { 0x0958, 0x0961, 1}, |
| { 0x0971, 0x0980, 1}, |
| { 0x0985, 0x098c, 1}, |
| { 0x098f, 0x0990, 1}, |
| { 0x0993, 0x09a8, 1}, |
| { 0x09aa, 0x09b0, 1}, |
| { 0x09b2, 0x09b6, 4}, |
| { 0x09b7, 0x09b9, 1}, |
| { 0x09bd, 0x09ce, 17}, |
| { 0x09dc, 0x09dd, 1}, |
| { 0x09df, 0x09e1, 1}, |
| { 0x09f0, 0x09f1, 1}, |
| { 0x0a05, 0x0a0a, 1}, |
| { 0x0a0f, 0x0a10, 1}, |
| { 0x0a13, 0x0a28, 1}, |
| { 0x0a2a, 0x0a30, 1}, |
| { 0x0a32, 0x0a33, 1}, |
| { 0x0a35, 0x0a36, 1}, |
| { 0x0a38, 0x0a39, 1}, |
| { 0x0a59, 0x0a5c, 1}, |
| { 0x0a5e, 0x0a72, 20}, |
| { 0x0a73, 0x0a74, 1}, |
| { 0x0a85, 0x0a8d, 1}, |
| { 0x0a8f, 0x0a91, 1}, |
| { 0x0a93, 0x0aa8, 1}, |
| { 0x0aaa, 0x0ab0, 1}, |
| { 0x0ab2, 0x0ab3, 1}, |
| { 0x0ab5, 0x0ab9, 1}, |
| { 0x0abd, 0x0ad0, 19}, |
| { 0x0ae0, 0x0ae1, 1}, |
| { 0x0af9, 0x0b05, 12}, |
| { 0x0b06, 0x0b0c, 1}, |
| { 0x0b0f, 0x0b10, 1}, |
| { 0x0b13, 0x0b28, 1}, |
| { 0x0b2a, 0x0b30, 1}, |
| { 0x0b32, 0x0b33, 1}, |
| { 0x0b35, 0x0b39, 1}, |
| { 0x0b3d, 0x0b5c, 31}, |
| { 0x0b5d, 0x0b5f, 2}, |
| { 0x0b60, 0x0b61, 1}, |
| { 0x0b71, 0x0b83, 18}, |
| { 0x0b85, 0x0b8a, 1}, |
| { 0x0b8e, 0x0b90, 1}, |
| { 0x0b92, 0x0b95, 1}, |
| { 0x0b99, 0x0b9a, 1}, |
| { 0x0b9c, 0x0b9e, 2}, |
| { 0x0b9f, 0x0ba3, 4}, |
| { 0x0ba4, 0x0ba8, 4}, |
| { 0x0ba9, 0x0baa, 1}, |
| { 0x0bae, 0x0bb9, 1}, |
| { 0x0bd0, 0x0c05, 53}, |
| { 0x0c06, 0x0c0c, 1}, |
| { 0x0c0e, 0x0c10, 1}, |
| { 0x0c12, 0x0c28, 1}, |
| { 0x0c2a, 0x0c39, 1}, |
| { 0x0c3d, 0x0c58, 27}, |
| { 0x0c59, 0x0c5a, 1}, |
| { 0x0c60, 0x0c61, 1}, |
| { 0x0c85, 0x0c8c, 1}, |
| { 0x0c8e, 0x0c90, 1}, |
| { 0x0c92, 0x0ca8, 1}, |
| { 0x0caa, 0x0cb3, 1}, |
| { 0x0cb5, 0x0cb9, 1}, |
| { 0x0cbd, 0x0cde, 33}, |
| { 0x0ce0, 0x0ce1, 1}, |
| { 0x0cf1, 0x0cf2, 1}, |
| { 0x0d05, 0x0d0c, 1}, |
| { 0x0d0e, 0x0d10, 1}, |
| { 0x0d12, 0x0d3a, 1}, |
| { 0x0d3d, 0x0d5f, 17}, |
| { 0x0d60, 0x0d61, 1}, |
| { 0x0d7a, 0x0d7f, 1}, |
| { 0x0d85, 0x0d96, 1}, |
| { 0x0d9a, 0x0db1, 1}, |
| { 0x0db3, 0x0dbb, 1}, |
| { 0x0dbd, 0x0dc0, 3}, |
| { 0x0dc1, 0x0dc6, 1}, |
| { 0x0e01, 0x0e30, 1}, |
| { 0x0e32, 0x0e33, 1}, |
| { 0x0e40, 0x0e46, 1}, |
| { 0x0e81, 0x0e82, 1}, |
| { 0x0e84, 0x0e87, 3}, |
| { 0x0e88, 0x0e8a, 2}, |
| { 0x0e8d, 0x0e94, 7}, |
| { 0x0e95, 0x0e97, 1}, |
| { 0x0e99, 0x0e9f, 1}, |
| { 0x0ea1, 0x0ea3, 1}, |
| { 0x0ea5, 0x0ea7, 2}, |
| { 0x0eaa, 0x0eab, 1}, |
| { 0x0ead, 0x0eb0, 1}, |
| { 0x0eb2, 0x0eb3, 1}, |
| { 0x0ebd, 0x0ec0, 3}, |
| { 0x0ec1, 0x0ec4, 1}, |
| { 0x0ec6, 0x0edc, 22}, |
| { 0x0edd, 0x0edf, 1}, |
| { 0x0f00, 0x0f40, 64}, |
| { 0x0f41, 0x0f47, 1}, |
| { 0x0f49, 0x0f6c, 1}, |
| { 0x0f88, 0x0f8c, 1}, |
| { 0x1000, 0x102a, 1}, |
| { 0x103f, 0x1050, 17}, |
| { 0x1051, 0x1055, 1}, |
| { 0x105a, 0x105d, 1}, |
| { 0x1061, 0x1065, 4}, |
| { 0x1066, 0x106e, 8}, |
| { 0x106f, 0x1070, 1}, |
| { 0x1075, 0x1081, 1}, |
| { 0x108e, 0x10a0, 18}, |
| { 0x10a1, 0x10c5, 1}, |
| { 0x10c7, 0x10cd, 6}, |
| { 0x10d0, 0x10fa, 1}, |
| { 0x10fc, 0x1248, 1}, |
| { 0x124a, 0x124d, 1}, |
| { 0x1250, 0x1256, 1}, |
| { 0x1258, 0x125a, 2}, |
| { 0x125b, 0x125d, 1}, |
| { 0x1260, 0x1288, 1}, |
| { 0x128a, 0x128d, 1}, |
| { 0x1290, 0x12b0, 1}, |
| { 0x12b2, 0x12b5, 1}, |
| { 0x12b8, 0x12be, 1}, |
| { 0x12c0, 0x12c2, 2}, |
| { 0x12c3, 0x12c5, 1}, |
| { 0x12c8, 0x12d6, 1}, |
| { 0x12d8, 0x1310, 1}, |
| { 0x1312, 0x1315, 1}, |
| { 0x1318, 0x135a, 1}, |
| { 0x1380, 0x138f, 1}, |
| { 0x13a0, 0x13f5, 1}, |
| { 0x13f8, 0x13fd, 1}, |
| { 0x1401, 0x166c, 1}, |
| { 0x166f, 0x167f, 1}, |
| { 0x1681, 0x169a, 1}, |
| { 0x16a0, 0x16ea, 1}, |
| { 0x16f1, 0x16f8, 1}, |
| { 0x1700, 0x170c, 1}, |
| { 0x170e, 0x1711, 1}, |
| { 0x1720, 0x1731, 1}, |
| { 0x1740, 0x1751, 1}, |
| { 0x1760, 0x176c, 1}, |
| { 0x176e, 0x1770, 1}, |
| { 0x1780, 0x17b3, 1}, |
| { 0x17d7, 0x17dc, 5}, |
| { 0x1820, 0x1877, 1}, |
| { 0x1880, 0x18a8, 1}, |
| { 0x18aa, 0x18b0, 6}, |
| { 0x18b1, 0x18f5, 1}, |
| { 0x1900, 0x191e, 1}, |
| { 0x1950, 0x196d, 1}, |
| { 0x1970, 0x1974, 1}, |
| { 0x1980, 0x19ab, 1}, |
| { 0x19b0, 0x19c9, 1}, |
| { 0x1a00, 0x1a16, 1}, |
| { 0x1a20, 0x1a54, 1}, |
| { 0x1aa7, 0x1b05, 94}, |
| { 0x1b06, 0x1b33, 1}, |
| { 0x1b45, 0x1b4b, 1}, |
| { 0x1b83, 0x1ba0, 1}, |
| { 0x1bae, 0x1baf, 1}, |
| { 0x1bba, 0x1be5, 1}, |
| { 0x1c00, 0x1c23, 1}, |
| { 0x1c4d, 0x1c4f, 1}, |
| { 0x1c5a, 0x1c7d, 1}, |
| { 0x1ce9, 0x1cec, 1}, |
| { 0x1cee, 0x1cf1, 1}, |
| { 0x1cf5, 0x1cf6, 1}, |
| { 0x1d00, 0x1dbf, 1}, |
| { 0x1e00, 0x1f15, 1}, |
| { 0x1f18, 0x1f1d, 1}, |
| { 0x1f20, 0x1f45, 1}, |
| { 0x1f48, 0x1f4d, 1}, |
| { 0x1f50, 0x1f57, 1}, |
| { 0x1f59, 0x1f5f, 2}, |
| { 0x1f60, 0x1f7d, 1}, |
| { 0x1f80, 0x1fb4, 1}, |
| { 0x1fb6, 0x1fbc, 1}, |
| { 0x1fbe, 0x1fc2, 4}, |
| { 0x1fc3, 0x1fc4, 1}, |
| { 0x1fc6, 0x1fcc, 1}, |
| { 0x1fd0, 0x1fd3, 1}, |
| { 0x1fd6, 0x1fdb, 1}, |
| { 0x1fe0, 0x1fec, 1}, |
| { 0x1ff2, 0x1ff4, 1}, |
| { 0x1ff6, 0x1ffc, 1}, |
| { 0x2071, 0x207f, 14}, |
| { 0x2090, 0x209c, 1}, |
| { 0x2102, 0x2107, 5}, |
| { 0x210a, 0x2113, 1}, |
| { 0x2115, 0x2119, 4}, |
| { 0x211a, 0x211d, 1}, |
| { 0x2124, 0x212a, 2}, |
| { 0x212b, 0x212d, 1}, |
| { 0x212f, 0x2139, 1}, |
| { 0x213c, 0x213f, 1}, |
| { 0x2145, 0x2149, 1}, |
| { 0x214e, 0x2183, 53}, |
| { 0x2184, 0x2c00, 2684}, |
| { 0x2c01, 0x2c2e, 1}, |
| { 0x2c30, 0x2c5e, 1}, |
| { 0x2c60, 0x2ce4, 1}, |
| { 0x2ceb, 0x2cee, 1}, |
| { 0x2cf2, 0x2cf3, 1}, |
| { 0x2d00, 0x2d25, 1}, |
| { 0x2d27, 0x2d2d, 6}, |
| { 0x2d30, 0x2d67, 1}, |
| { 0x2d6f, 0x2d80, 17}, |
| { 0x2d81, 0x2d96, 1}, |
| { 0x2da0, 0x2da6, 1}, |
| { 0x2da8, 0x2dae, 1}, |
| { 0x2db0, 0x2db6, 1}, |
| { 0x2db8, 0x2dbe, 1}, |
| { 0x2dc0, 0x2dc6, 1}, |
| { 0x2dc8, 0x2dce, 1}, |
| { 0x2dd0, 0x2dd6, 1}, |
| { 0x2dd8, 0x2dde, 1}, |
| { 0x2e2f, 0x3005, 470}, |
| { 0x3006, 0x3031, 43}, |
| { 0x3032, 0x3035, 1}, |
| { 0x303b, 0x303c, 1}, |
| { 0x3041, 0x3096, 1}, |
| { 0x309d, 0x309f, 1}, |
| { 0x30a1, 0x30fa, 1}, |
| { 0x30fc, 0x30ff, 1}, |
| { 0x3105, 0x312d, 1}, |
| { 0x3131, 0x318e, 1}, |
| { 0x31a0, 0x31ba, 1}, |
| { 0x31f0, 0x31ff, 1}, |
| { 0x3400, 0x4db5, 1}, |
| { 0x4e00, 0x9fd5, 1}, |
| { 0xa000, 0xa48c, 1}, |
| { 0xa4d0, 0xa4fd, 1}, |
| { 0xa500, 0xa60c, 1}, |
| { 0xa610, 0xa61f, 1}, |
| { 0xa62a, 0xa62b, 1}, |
| { 0xa640, 0xa66e, 1}, |
| { 0xa67f, 0xa69d, 1}, |
| { 0xa6a0, 0xa6e5, 1}, |
| { 0xa717, 0xa71f, 1}, |
| { 0xa722, 0xa788, 1}, |
| { 0xa78b, 0xa7ad, 1}, |
| { 0xa7b0, 0xa7b7, 1}, |
| { 0xa7f7, 0xa801, 1}, |
| { 0xa803, 0xa805, 1}, |
| { 0xa807, 0xa80a, 1}, |
| { 0xa80c, 0xa822, 1}, |
| { 0xa840, 0xa873, 1}, |
| { 0xa882, 0xa8b3, 1}, |
| { 0xa8f2, 0xa8f7, 1}, |
| { 0xa8fb, 0xa8fd, 2}, |
| { 0xa90a, 0xa925, 1}, |
| { 0xa930, 0xa946, 1}, |
| { 0xa960, 0xa97c, 1}, |
| { 0xa984, 0xa9b2, 1}, |
| { 0xa9cf, 0xa9e0, 17}, |
| { 0xa9e1, 0xa9e4, 1}, |
| { 0xa9e6, 0xa9ef, 1}, |
| { 0xa9fa, 0xa9fe, 1}, |
| { 0xaa00, 0xaa28, 1}, |
| { 0xaa40, 0xaa42, 1}, |
| { 0xaa44, 0xaa4b, 1}, |
| { 0xaa60, 0xaa76, 1}, |
| { 0xaa7a, 0xaa7e, 4}, |
| { 0xaa7f, 0xaaaf, 1}, |
| { 0xaab1, 0xaab5, 4}, |
| { 0xaab6, 0xaab9, 3}, |
| { 0xaaba, 0xaabd, 1}, |
| { 0xaac0, 0xaac2, 2}, |
| { 0xaadb, 0xaadd, 1}, |
| { 0xaae0, 0xaaea, 1}, |
| { 0xaaf2, 0xaaf4, 1}, |
| { 0xab01, 0xab06, 1}, |
| { 0xab09, 0xab0e, 1}, |
| { 0xab11, 0xab16, 1}, |
| { 0xab20, 0xab26, 1}, |
| { 0xab28, 0xab2e, 1}, |
| { 0xab30, 0xab5a, 1}, |
| { 0xab5c, 0xab65, 1}, |
| { 0xab70, 0xabe2, 1}, |
| { 0xac00, 0xd7a3, 1}, |
| { 0xd7b0, 0xd7c6, 1}, |
| { 0xd7cb, 0xd7fb, 1}, |
| { 0xf900, 0xfa6d, 1}, |
| { 0xfa70, 0xfad9, 1}, |
| { 0xfb00, 0xfb06, 1}, |
| { 0xfb13, 0xfb17, 1}, |
| { 0xfb1d, 0xfb1f, 2}, |
| { 0xfb20, 0xfb28, 1}, |
| { 0xfb2a, 0xfb36, 1}, |
| { 0xfb38, 0xfb3c, 1}, |
| { 0xfb3e, 0xfb40, 2}, |
| { 0xfb41, 0xfb43, 2}, |
| { 0xfb44, 0xfb46, 2}, |
| { 0xfb47, 0xfbb1, 1}, |
| { 0xfbd3, 0xfd3d, 1}, |
| { 0xfd50, 0xfd8f, 1}, |
| { 0xfd92, 0xfdc7, 1}, |
| { 0xfdf0, 0xfdfb, 1}, |
| { 0xfe70, 0xfe74, 1}, |
| { 0xfe76, 0xfefc, 1}, |
| { 0xff21, 0xff3a, 1}, |
| { 0xff41, 0xff5a, 1}, |
| { 0xff66, 0xffbe, 1}, |
| { 0xffc2, 0xffc7, 1}, |
| { 0xffca, 0xffcf, 1}, |
| { 0xffd2, 0xffd7, 1}, |
| { 0xffda, 0xffdc, 1}, |
| { 0x10000, 0x1000b, 1}, |
| { 0x1000d, 0x10026, 1}, |
| { 0x10028, 0x1003a, 1}, |
| { 0x1003c, 0x1003d, 1}, |
| { 0x1003f, 0x1004d, 1}, |
| { 0x10050, 0x1005d, 1}, |
| { 0x10080, 0x100fa, 1}, |
| { 0x10280, 0x1029c, 1}, |
| { 0x102a0, 0x102d0, 1}, |
| { 0x10300, 0x1031f, 1}, |
| { 0x10330, 0x10340, 1}, |
| { 0x10342, 0x10349, 1}, |
| { 0x10350, 0x10375, 1}, |
| { 0x10380, 0x1039d, 1}, |
| { 0x103a0, 0x103c3, 1}, |
| { 0x103c8, 0x103cf, 1}, |
| { 0x10400, 0x1049d, 1}, |
| { 0x10500, 0x10527, 1}, |
| { 0x10530, 0x10563, 1}, |
| { 0x10600, 0x10736, 1}, |
| { 0x10740, 0x10755, 1}, |
| { 0x10760, 0x10767, 1}, |
| { 0x10800, 0x10805, 1}, |
| { 0x10808, 0x1080a, 2}, |
| { 0x1080b, 0x10835, 1}, |
| { 0x10837, 0x10838, 1}, |
| { 0x1083c, 0x1083f, 3}, |
| { 0x10840, 0x10855, 1}, |
| { 0x10860, 0x10876, 1}, |
| { 0x10880, 0x1089e, 1}, |
| { 0x108e0, 0x108f2, 1}, |
| { 0x108f4, 0x108f5, 1}, |
| { 0x10900, 0x10915, 1}, |
| { 0x10920, 0x10939, 1}, |
| { 0x10980, 0x109b7, 1}, |
| { 0x109be, 0x109bf, 1}, |
| { 0x10a00, 0x10a10, 16}, |
| { 0x10a11, 0x10a13, 1}, |
| { 0x10a15, 0x10a17, 1}, |
| { 0x10a19, 0x10a33, 1}, |
| { 0x10a60, 0x10a7c, 1}, |
| { 0x10a80, 0x10a9c, 1}, |
| { 0x10ac0, 0x10ac7, 1}, |
| { 0x10ac9, 0x10ae4, 1}, |
| { 0x10b00, 0x10b35, 1}, |
| { 0x10b40, 0x10b55, 1}, |
| { 0x10b60, 0x10b72, 1}, |
| { 0x10b80, 0x10b91, 1}, |
| { 0x10c00, 0x10c48, 1}, |
| { 0x10c80, 0x10cb2, 1}, |
| { 0x10cc0, 0x10cf2, 1}, |
| { 0x11003, 0x11037, 1}, |
| { 0x11083, 0x110af, 1}, |
| { 0x110d0, 0x110e8, 1}, |
| { 0x11103, 0x11126, 1}, |
| { 0x11150, 0x11172, 1}, |
| { 0x11176, 0x11183, 13}, |
| { 0x11184, 0x111b2, 1}, |
| { 0x111c1, 0x111c4, 1}, |
| { 0x111da, 0x111dc, 2}, |
| { 0x11200, 0x11211, 1}, |
| { 0x11213, 0x1122b, 1}, |
| { 0x11280, 0x11286, 1}, |
| { 0x11288, 0x1128a, 2}, |
| { 0x1128b, 0x1128d, 1}, |
| { 0x1128f, 0x1129d, 1}, |
| { 0x1129f, 0x112a8, 1}, |
| { 0x112b0, 0x112de, 1}, |
| { 0x11305, 0x1130c, 1}, |
| { 0x1130f, 0x11310, 1}, |
| { 0x11313, 0x11328, 1}, |
| { 0x1132a, 0x11330, 1}, |
| { 0x11332, 0x11333, 1}, |
| { 0x11335, 0x11339, 1}, |
| { 0x1133d, 0x11350, 19}, |
| { 0x1135d, 0x11361, 1}, |
| { 0x11480, 0x114af, 1}, |
| { 0x114c4, 0x114c5, 1}, |
| { 0x114c7, 0x11580, 185}, |
| { 0x11581, 0x115ae, 1}, |
| { 0x115d8, 0x115db, 1}, |
| { 0x11600, 0x1162f, 1}, |
| { 0x11644, 0x11680, 60}, |
| { 0x11681, 0x116aa, 1}, |
| { 0x11700, 0x11719, 1}, |
| { 0x118a0, 0x118df, 1}, |
| { 0x118ff, 0x11ac0, 449}, |
| { 0x11ac1, 0x11af8, 1}, |
| { 0x12000, 0x12399, 1}, |
| { 0x12480, 0x12543, 1}, |
| { 0x13000, 0x1342e, 1}, |
| { 0x14400, 0x14646, 1}, |
| { 0x16800, 0x16a38, 1}, |
| { 0x16a40, 0x16a5e, 1}, |
| { 0x16ad0, 0x16aed, 1}, |
| { 0x16b00, 0x16b2f, 1}, |
| { 0x16b40, 0x16b43, 1}, |
| { 0x16b63, 0x16b77, 1}, |
| { 0x16b7d, 0x16b8f, 1}, |
| { 0x16f00, 0x16f44, 1}, |
| { 0x16f50, 0x16f93, 67}, |
| { 0x16f94, 0x16f9f, 1}, |
| { 0x1b000, 0x1b001, 1}, |
| { 0x1bc00, 0x1bc6a, 1}, |
| { 0x1bc70, 0x1bc7c, 1}, |
| { 0x1bc80, 0x1bc88, 1}, |
| { 0x1bc90, 0x1bc99, 1}, |
| { 0x1d400, 0x1d454, 1}, |
| { 0x1d456, 0x1d49c, 1}, |
| { 0x1d49e, 0x1d49f, 1}, |
| { 0x1d4a2, 0x1d4a5, 3}, |
| { 0x1d4a6, 0x1d4a9, 3}, |
| { 0x1d4aa, 0x1d4ac, 1}, |
| { 0x1d4ae, 0x1d4b9, 1}, |
| { 0x1d4bb, 0x1d4bd, 2}, |
| { 0x1d4be, 0x1d4c3, 1}, |
| { 0x1d4c5, 0x1d505, 1}, |
| { 0x1d507, 0x1d50a, 1}, |
| { 0x1d50d, 0x1d514, 1}, |
| { 0x1d516, 0x1d51c, 1}, |
| { 0x1d51e, 0x1d539, 1}, |
| { 0x1d53b, 0x1d53e, 1}, |
| { 0x1d540, 0x1d544, 1}, |
| { 0x1d546, 0x1d54a, 4}, |
| { 0x1d54b, 0x1d550, 1}, |
| { 0x1d552, 0x1d6a5, 1}, |
| { 0x1d6a8, 0x1d6c0, 1}, |
| { 0x1d6c2, 0x1d6da, 1}, |
| { 0x1d6dc, 0x1d6fa, 1}, |
| { 0x1d6fc, 0x1d714, 1}, |
| { 0x1d716, 0x1d734, 1}, |
| { 0x1d736, 0x1d74e, 1}, |
| { 0x1d750, 0x1d76e, 1}, |
| { 0x1d770, 0x1d788, 1}, |
| { 0x1d78a, 0x1d7a8, 1}, |
| { 0x1d7aa, 0x1d7c2, 1}, |
| { 0x1d7c4, 0x1d7cb, 1}, |
| { 0x1e800, 0x1e8c4, 1}, |
| { 0x1ee00, 0x1ee03, 1}, |
| { 0x1ee05, 0x1ee1f, 1}, |
| { 0x1ee21, 0x1ee22, 1}, |
| { 0x1ee24, 0x1ee27, 3}, |
| { 0x1ee29, 0x1ee32, 1}, |
| { 0x1ee34, 0x1ee37, 1}, |
| { 0x1ee39, 0x1ee3b, 2}, |
| { 0x1ee42, 0x1ee47, 5}, |
| { 0x1ee49, 0x1ee4d, 2}, |
| { 0x1ee4e, 0x1ee4f, 1}, |
| { 0x1ee51, 0x1ee52, 1}, |
| { 0x1ee54, 0x1ee57, 3}, |
| { 0x1ee59, 0x1ee61, 2}, |
| { 0x1ee62, 0x1ee64, 2}, |
| { 0x1ee67, 0x1ee6a, 1}, |
| { 0x1ee6c, 0x1ee72, 1}, |
| { 0x1ee74, 0x1ee77, 1}, |
| { 0x1ee79, 0x1ee7c, 1}, |
| { 0x1ee7e, 0x1ee80, 2}, |
| { 0x1ee81, 0x1ee89, 1}, |
| { 0x1ee8b, 0x1ee9b, 1}, |
| { 0x1eea1, 0x1eea3, 1}, |
| { 0x1eea5, 0x1eea9, 1}, |
| { 0x1eeab, 0x1eebb, 1}, |
| { 0x20000, 0x2a6d6, 1}, |
| |