internal/export/unicode: copy unicode/maketables.go
copying core's unicode/maketables.go into x/text
in preparation for rework
Copied from master at
26985ed4a58665d25a256e3b63b353972fc3aab0
Change-Id: I343366e69081fb2dbd6030b403386aeae1b0f6d8
Reviewed-on: https://go-review.googlesource.com/c/154437
Run-TryBot: Marcel van Lohuizen <mpvl@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Ross Light <light@google.com>
diff --git a/internal/export/unicode/maketables.go b/internal/export/unicode/maketables.go
new file mode 100644
index 0000000..a1f1586
--- /dev/null
+++ b/internal/export/unicode/maketables.go
@@ -0,0 +1,1452 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Unicode table generator.
+// Data read from the web.
+
+package main
+
+import (
+ "bufio"
+ "flag"
+ "fmt"
+ "io"
+ "log"
+ "net/http"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "sort"
+ "strconv"
+ "strings"
+ "unicode"
+)
+
+func main() {
+ flag.Parse()
+ setupOutput()
+ loadChars() // always needed
+ loadCasefold()
+ printCategories()
+ printScriptOrProperty(false)
+ printScriptOrProperty(true)
+ printCases()
+ printLatinProperties()
+ printCasefold()
+ printSizes()
+ flushOutput()
+}
+
+func defaultVersion() string {
+ if v := os.Getenv("UNICODE_VERSION"); v != "" {
+ return v
+ }
+ return unicode.Version
+}
+
+var dataURL = flag.String("data", "", "full URL for UnicodeData.txt; defaults to --url/UnicodeData.txt")
+var casefoldingURL = flag.String("casefolding", "", "full URL for CaseFolding.txt; defaults to --url/CaseFolding.txt")
+var url = flag.String("url",
+ "http://www.unicode.org/Public/"+defaultVersion()+"/ucd/",
+ "URL of Unicode database directory")
+var tablelist = flag.String("tables",
+ "all",
+ "comma-separated list of which tables to generate; can be letter")
+var scriptlist = flag.String("scripts",
+ "all",
+ "comma-separated list of which script tables to generate")
+var proplist = flag.String("props",
+ "all",
+ "comma-separated list of which property tables to generate")
+var cases = flag.Bool("cases",
+ true,
+ "generate case tables")
+var test = flag.Bool("test",
+ false,
+ "test existing tables; can be used to compare web data with package data")
+var localFiles = flag.Bool("local",
+ false,
+ "data files have been copied to current directory; for debugging only")
+var outputFile = flag.String("output",
+ "",
+ "output file for generated tables; default stdout")
+
+var scriptRe = regexp.MustCompile(`^([0-9A-F]+)(\.\.[0-9A-F]+)? *; ([A-Za-z_]+)$`)
+var logger = log.New(os.Stderr, "", log.Lshortfile)
+
+var output *bufio.Writer // points to os.Stdout or to "gofmt > outputFile"
+
+func setupOutput() {
+ output = bufio.NewWriter(startGofmt())
+}
+
+// startGofmt connects output to a gofmt process if -output is set.
+func startGofmt() io.Writer {
+ if *outputFile == "" {
+ return os.Stdout
+ }
+ stdout, err := os.Create(*outputFile)
+ if err != nil {
+ logger.Fatal(err)
+ }
+ // Pipe output to gofmt.
+ gofmt := exec.Command("gofmt")
+ fd, err := gofmt.StdinPipe()
+ if err != nil {
+ logger.Fatal(err)
+ }
+ gofmt.Stdout = stdout
+ gofmt.Stderr = os.Stderr
+ err = gofmt.Start()
+ if err != nil {
+ logger.Fatal(err)
+ }
+ return fd
+}
+
+func flushOutput() {
+ err := output.Flush()
+ if err != nil {
+ logger.Fatal(err)
+ }
+}
+
+func printf(format string, args ...interface{}) {
+ fmt.Fprintf(output, format, args...)
+}
+
+func print(args ...interface{}) {
+ fmt.Fprint(output, args...)
+}
+
+func println(args ...interface{}) {
+ fmt.Fprintln(output, args...)
+}
+
+type reader struct {
+ *bufio.Reader
+ fd *os.File
+ resp *http.Response
+}
+
+func open(url string) *reader {
+ file := filepath.Base(url)
+ if *localFiles {
+ fd, err := os.Open(file)
+ if err != nil {
+ logger.Fatal(err)
+ }
+ return &reader{bufio.NewReader(fd), fd, nil}
+ }
+ resp, err := http.Get(url)
+ if err != nil {
+ logger.Fatal(err)
+ }
+ if resp.StatusCode != 200 {
+ logger.Fatalf("bad GET status for %s: %d", file, resp.Status)
+ }
+ return &reader{bufio.NewReader(resp.Body), nil, resp}
+
+}
+
+func (r *reader) close() {
+ if r.fd != nil {
+ r.fd.Close()
+ } else {
+ r.resp.Body.Close()
+ }
+}
+
+var category = map[string]bool{
+ // Nd Lu etc.
+ // We use one-character names to identify merged categories
+ "L": true, // Lu Ll Lt Lm Lo
+ "P": true, // Pc Pd Ps Pe Pu Pf Po
+ "M": true, // Mn Mc Me
+ "N": true, // Nd Nl No
+ "S": true, // Sm Sc Sk So
+ "Z": true, // Zs Zl Zp
+ "C": true, // Cc Cf Cs Co Cn
+}
+
+// UnicodeData.txt has form:
+// 0037;DIGIT SEVEN;Nd;0;EN;;7;7;7;N;;;;;
+// 007A;LATIN SMALL LETTER Z;Ll;0;L;;;;;N;;;005A;;005A
+// See https://www.unicode.org/reports/tr44/ for a full explanation
+// The fields:
+const (
+ FCodePoint = iota
+ FName
+ FGeneralCategory
+ FCanonicalCombiningClass
+ FBidiClass
+ FDecompositionTypeAndMapping
+ FNumericType
+ FNumericDigit // If a decimal digit.
+ FNumericValue // Includes non-decimal, e.g. U+2155=1/5
+ FBidiMirrored
+ FUnicode1Name
+ FISOComment
+ FSimpleUppercaseMapping
+ FSimpleLowercaseMapping
+ FSimpleTitlecaseMapping
+ NumField
+
+ MaxChar = 0x10FFFF // anything above this shouldn't exist
+)
+
+var fieldName = []string{
+ FCodePoint: "CodePoint",
+ FName: "Name",
+ FGeneralCategory: "GeneralCategory",
+ FCanonicalCombiningClass: "CanonicalCombiningClass",
+ FBidiClass: "BidiClass",
+ FDecompositionTypeAndMapping: "DecompositionTypeAndMapping",
+ FNumericType: "NumericType",
+ FNumericDigit: "NumericDigit",
+ FNumericValue: "NumericValue",
+ FBidiMirrored: "BidiMirrored",
+ FUnicode1Name: "Unicode1Name",
+ FISOComment: "ISOComment",
+ FSimpleUppercaseMapping: "SimpleUppercaseMapping",
+ FSimpleLowercaseMapping: "SimpleLowercaseMapping",
+ FSimpleTitlecaseMapping: "SimpleTitlecaseMapping",
+}
+
+// This contains only the properties we're interested in.
+type Char struct {
+ field []string // debugging only; could be deleted if we take out char.dump()
+ codePoint rune // if zero, this index is not a valid code point.
+ category string
+ upperCase rune
+ lowerCase rune
+ titleCase rune
+ foldCase rune // simple case folding
+ caseOrbit rune // next in simple case folding orbit
+}
+
+// Scripts.txt has form:
+// A673 ; Cyrillic # Po SLAVONIC ASTERISK
+// A67C..A67D ; Cyrillic # Mn [2] COMBINING CYRILLIC KAVYKA..COMBINING CYRILLIC PAYEROK
+// See https://www.unicode.org/Public/5.1.0/ucd/UCD.html for full explanation
+
+type Script struct {
+ lo, hi uint32 // range of code points
+ script string
+}
+
+var chars = make([]Char, MaxChar+1)
+var scripts = make(map[string][]Script)
+var props = make(map[string][]Script) // a property looks like a script; can share the format
+
+var lastChar rune = 0
+
+// In UnicodeData.txt, some ranges are marked like this:
+// 3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
+// 4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
+// parseCategory returns a state variable indicating the weirdness.
+type State int
+
+const (
+ SNormal State = iota // known to be zero for the type
+ SFirst
+ SLast
+ SMissing
+)
+
+func parseCategory(line string) (state State) {
+ field := strings.Split(line, ";")
+ if len(field) != NumField {
+ logger.Fatalf("%5s: %d fields (expected %d)\n", line, len(field), NumField)
+ }
+ point, err := strconv.ParseUint(field[FCodePoint], 16, 64)
+ if err != nil {
+ logger.Fatalf("%.5s...: %s", line, err)
+ }
+ lastChar = rune(point)
+ if point > MaxChar {
+ return
+ }
+ char := &chars[point]
+ char.field = field
+ if char.codePoint != 0 {
+ logger.Fatalf("point %U reused", point)
+ }
+ char.codePoint = lastChar
+ char.category = field[FGeneralCategory]
+ category[char.category] = true
+ switch char.category {
+ case "Nd":
+ // Decimal digit
+ _, err := strconv.Atoi(field[FNumericValue])
+ if err != nil {
+ logger.Fatalf("%U: bad numeric field: %s", point, err)
+ }
+ case "Lu":
+ char.letter(field[FCodePoint], field[FSimpleLowercaseMapping], field[FSimpleTitlecaseMapping])
+ case "Ll":
+ char.letter(field[FSimpleUppercaseMapping], field[FCodePoint], field[FSimpleTitlecaseMapping])
+ case "Lt":
+ char.letter(field[FSimpleUppercaseMapping], field[FSimpleLowercaseMapping], field[FCodePoint])
+ default:
+ char.letter(field[FSimpleUppercaseMapping], field[FSimpleLowercaseMapping], field[FSimpleTitlecaseMapping])
+ }
+ switch {
+ case strings.Index(field[FName], ", First>") > 0:
+ state = SFirst
+ case strings.Index(field[FName], ", Last>") > 0:
+ state = SLast
+ }
+ return
+}
+
+func (char *Char) dump(s string) {
+ print(s, " ")
+ for i := 0; i < len(char.field); i++ {
+ printf("%s:%q ", fieldName[i], char.field[i])
+ }
+ print("\n")
+}
+
+func (char *Char) letter(u, l, t string) {
+ char.upperCase = char.letterValue(u, "U")
+ char.lowerCase = char.letterValue(l, "L")
+ char.titleCase = char.letterValue(t, "T")
+}
+
+func (char *Char) letterValue(s string, cas string) rune {
+ if s == "" {
+ return 0
+ }
+ v, err := strconv.ParseUint(s, 16, 64)
+ if err != nil {
+ char.dump(cas)
+ logger.Fatalf("%U: bad letter(%s): %s", char.codePoint, s, err)
+ }
+ return rune(v)
+}
+
+func allCategories() []string {
+ a := make([]string, 0, len(category))
+ for k := range category {
+ a = append(a, k)
+ }
+ sort.Strings(a)
+ return a
+}
+
+func all(scripts map[string][]Script) []string {
+ a := make([]string, 0, len(scripts))
+ for k := range scripts {
+ a = append(a, k)
+ }
+ sort.Strings(a)
+ return a
+}
+
+func allCatFold(m map[string]map[rune]bool) []string {
+ a := make([]string, 0, len(m))
+ for k := range m {
+ a = append(a, k)
+ }
+ sort.Strings(a)
+ return a
+}
+
+// Extract the version number from the URL
+func version() string {
+ // Break on slashes and look for the first numeric field
+ fields := strings.Split(*url, "/")
+ for _, f := range fields {
+ if len(f) > 0 && '0' <= f[0] && f[0] <= '9' {
+ return f
+ }
+ }
+ logger.Fatal("unknown version")
+ return "Unknown"
+}
+
+func categoryOp(code rune, class uint8) bool {
+ category := chars[code].category
+ return len(category) > 0 && category[0] == class
+}
+
+func loadChars() {
+ if *dataURL == "" {
+ flag.Set("data", *url+"UnicodeData.txt")
+ }
+ input := open(*dataURL)
+ defer input.close()
+ scanner := bufio.NewScanner(input)
+ var first rune = 0
+ for scanner.Scan() {
+ switch parseCategory(scanner.Text()) {
+ case SNormal:
+ if first != 0 {
+ logger.Fatalf("bad state normal at %U", lastChar)
+ }
+ case SFirst:
+ if first != 0 {
+ logger.Fatalf("bad state first at %U", lastChar)
+ }
+ first = lastChar
+ case SLast:
+ if first == 0 {
+ logger.Fatalf("bad state last at %U", lastChar)
+ }
+ for i := first + 1; i <= lastChar; i++ {
+ chars[i] = chars[first]
+ chars[i].codePoint = i
+ }
+ first = 0
+ }
+ }
+ if scanner.Err() != nil {
+ logger.Fatal(scanner.Err())
+ }
+}
+
+func loadCasefold() {
+ if *casefoldingURL == "" {
+ flag.Set("casefolding", *url+"CaseFolding.txt")
+ }
+ input := open(*casefoldingURL)
+ defer input.close()
+ scanner := bufio.NewScanner(input)
+ for scanner.Scan() {
+ line := scanner.Text()
+ if len(line) == 0 || line[0] == '#' || len(strings.TrimSpace(line)) == 0 {
+ continue
+ }
+ field := strings.Split(line, "; ")
+ if len(field) != 4 {
+ logger.Fatalf("CaseFolding.txt %.5s...: %d fields (expected %d)\n", line, len(field), 4)
+ }
+ kind := field[1]
+ if kind != "C" && kind != "S" {
+ // Only care about 'common' and 'simple' foldings.
+ continue
+ }
+ p1, err := strconv.ParseUint(field[0], 16, 64)
+ if err != nil {
+ logger.Fatalf("CaseFolding.txt %.5s...: %s", line, err)
+ }
+ p2, err := strconv.ParseUint(field[2], 16, 64)
+ if err != nil {
+ logger.Fatalf("CaseFolding.txt %.5s...: %s", line, err)
+ }
+ chars[p1].foldCase = rune(p2)
+ }
+ if scanner.Err() != nil {
+ logger.Fatal(scanner.Err())
+ }
+}
+
+const progHeader = `// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by maketables; DO NOT EDIT.
+// To regenerate, run:
+// maketables --tables=%s --data=%s --casefolding=%s
+
+package unicode
+
+`
+
+var categoryMapping = map[string]string{
+ "Lu": "Letter, uppercase",
+ "Ll": "Letter, lowercase",
+ "Lt": "Letter, titlecase",
+ "Lm": "Letter, modifier",
+ "Lo": "Letter, other",
+ "Mn": "Mark, nonspacing",
+ "Mc": "Mark, spacing combining",
+ "Me": "Mark, enclosing",
+ "Nd": "Number, decimal digit",
+ "Nl": "Number, letter",
+ "No": "Number, other",
+ "Pc": "Punctuation, connector",
+ "Pd": "Punctuation, dash",
+ "Ps": "Punctuation, open",
+ "Pe": "Punctuation, close",
+ "Pi": "Punctuation, initial quote",
+ "Pf": "Punctuation, final quote",
+ "Po": "Punctuation, other",
+ "Sm": "Symbol, math",
+ "Sc": "Symbol, currency",
+ "Sk": "Symbol, modifier",
+ "So": "Symbol, other",
+ "Zs": "Separator, space",
+ "Zl": "Separator, line",
+ "Zp": "Separator, paragraph",
+ "Cc": "Other, control",
+ "Cf": "Other, format",
+ "Cs": "Other, surrogate",
+ "Co": "Other, private use",
+ "Cn": "Other, not assigned",
+}
+
+func printCategories() {
+ if *tablelist == "" {
+ return
+ }
+ // Find out which categories to dump
+ list := strings.Split(*tablelist, ",")
+ if *tablelist == "all" {
+ list = allCategories()
+ }
+ if *test {
+ fullCategoryTest(list)
+ return
+ }
+ printf(progHeader, *tablelist, *dataURL, *casefoldingURL)
+
+ println("// Version is the Unicode edition from which the tables are derived.")
+ printf("const Version = %q\n\n", version())
+
+ if *tablelist == "all" {
+ println("// Categories is the set of Unicode category tables.")
+ println("var Categories = map[string] *RangeTable {")
+ for _, k := range allCategories() {
+ printf("\t%q: %s,\n", k, k)
+ }
+ print("}\n\n")
+ }
+
+ decl := make(sort.StringSlice, len(list))
+ ndecl := 0
+ for _, name := range list {
+ if _, ok := category[name]; !ok {
+ logger.Fatal("unknown category", name)
+ }
+ // We generate an UpperCase name to serve as concise documentation and an _UnderScored
+ // name to store the data. This stops godoc dumping all the tables but keeps them
+ // available to clients.
+ // Cases deserving special comments
+ varDecl := ""
+ switch name {
+ case "C":
+ varDecl = "\tOther = _C; // Other/C is the set of Unicode control and special characters, category C.\n"
+ varDecl += "\tC = _C\n"
+ case "L":
+ varDecl = "\tLetter = _L; // Letter/L is the set of Unicode letters, category L.\n"
+ varDecl += "\tL = _L\n"
+ case "M":
+ varDecl = "\tMark = _M; // Mark/M is the set of Unicode mark characters, category M.\n"
+ varDecl += "\tM = _M\n"
+ case "N":
+ varDecl = "\tNumber = _N; // Number/N is the set of Unicode number characters, category N.\n"
+ varDecl += "\tN = _N\n"
+ case "P":
+ varDecl = "\tPunct = _P; // Punct/P is the set of Unicode punctuation characters, category P.\n"
+ varDecl += "\tP = _P\n"
+ case "S":
+ varDecl = "\tSymbol = _S; // Symbol/S is the set of Unicode symbol characters, category S.\n"
+ varDecl += "\tS = _S\n"
+ case "Z":
+ varDecl = "\tSpace = _Z; // Space/Z is the set of Unicode space characters, category Z.\n"
+ varDecl += "\tZ = _Z\n"
+ case "Nd":
+ varDecl = "\tDigit = _Nd; // Digit is the set of Unicode characters with the \"decimal digit\" property.\n"
+ case "Lu":
+ varDecl = "\tUpper = _Lu; // Upper is the set of Unicode upper case letters.\n"
+ case "Ll":
+ varDecl = "\tLower = _Ll; // Lower is the set of Unicode lower case letters.\n"
+ case "Lt":
+ varDecl = "\tTitle = _Lt; // Title is the set of Unicode title case letters.\n"
+ }
+ if len(name) > 1 {
+ desc, ok := categoryMapping[name]
+ if ok {
+ varDecl += fmt.Sprintf(
+ "\t%s = _%s; // %s is the set of Unicode characters in category %s (%s).\n",
+ name, name, name, name, desc)
+ } else {
+ varDecl += fmt.Sprintf(
+ "\t%s = _%s; // %s is the set of Unicode characters in category %s.\n",
+ name, name, name, name)
+ }
+ }
+ decl[ndecl] = varDecl
+ ndecl++
+ if len(name) == 1 { // unified categories
+ decl := fmt.Sprintf("var _%s = &RangeTable{\n", name)
+ dumpRange(
+ decl,
+ func(code rune) bool { return categoryOp(code, name[0]) })
+ continue
+ }
+ dumpRange(
+ fmt.Sprintf("var _%s = &RangeTable{\n", name),
+ func(code rune) bool { return chars[code].category == name })
+ }
+ decl.Sort()
+ println("// These variables have type *RangeTable.")
+ println("var (")
+ for _, d := range decl {
+ print(d)
+ }
+ print(")\n\n")
+}
+
+type Op func(code rune) bool
+
+const format = "\t\t{0x%04x, 0x%04x, %d},\n"
+
+func dumpRange(header string, inCategory Op) {
+ print(header)
+ next := rune(0)
+ latinOffset := 0
+ print("\tR16: []Range16{\n")
+ // one Range for each iteration
+ count := &range16Count
+ size := 16
+ for {
+ // look for start of range
+ for next < rune(len(chars)) && !inCategory(next) {
+ next++
+ }
+ if next >= rune(len(chars)) {
+ // no characters remain
+ break
+ }
+
+ // start of range
+ lo := next
+ hi := next
+ stride := rune(1)
+ // accept lo
+ next++
+ // look for another character to set the stride
+ for next < rune(len(chars)) && !inCategory(next) {
+ next++
+ }
+ if next >= rune(len(chars)) {
+ // no more characters
+ printf(format, lo, hi, stride)
+ break
+ }
+ // set stride
+ stride = next - lo
+ // check for length of run. next points to first jump in stride
+ for i := next; i < rune(len(chars)); i++ {
+ if inCategory(i) == (((i - lo) % stride) == 0) {
+ // accept
+ if inCategory(i) {
+ hi = i
+ }
+ } else {
+ // no more characters in this run
+ break
+ }
+ }
+ if uint32(hi) <= unicode.MaxLatin1 {
+ latinOffset++
+ }
+ size, count = printRange(uint32(lo), uint32(hi), uint32(stride), size, count)
+ // next range: start looking where this range ends
+ next = hi + 1
+ }
+ print("\t},\n")
+ if latinOffset > 0 {
+ printf("\tLatinOffset: %d,\n", latinOffset)
+ }
+ print("}\n\n")
+}
+
+func printRange(lo, hi, stride uint32, size int, count *int) (int, *int) {
+ if size == 16 && hi >= 1<<16 {
+ if lo < 1<<16 {
+ if lo+stride != hi {
+ logger.Fatalf("unexpected straddle: %U %U %d", lo, hi, stride)
+ }
+ // No range contains U+FFFF as an instance, so split
+ // the range into two entries. That way we can maintain
+ // the invariant that R32 contains only >= 1<<16.
+ printf(format, lo, lo, 1)
+ lo = hi
+ stride = 1
+ *count++
+ }
+ print("\t},\n")
+ print("\tR32: []Range32{\n")
+ size = 32
+ count = &range32Count
+ }
+ printf(format, lo, hi, stride)
+ *count++
+ return size, count
+}
+
+func fullCategoryTest(list []string) {
+ for _, name := range list {
+ if _, ok := category[name]; !ok {
+ logger.Fatal("unknown category", name)
+ }
+ r, ok := unicode.Categories[name]
+ if !ok && len(name) > 1 {
+ logger.Fatalf("unknown table %q", name)
+ }
+ if len(name) == 1 {
+ verifyRange(name, func(code rune) bool { return categoryOp(code, name[0]) }, r)
+ } else {
+ verifyRange(
+ name,
+ func(code rune) bool { return chars[code].category == name },
+ r)
+ }
+ }
+}
+
+func verifyRange(name string, inCategory Op, table *unicode.RangeTable) {
+ count := 0
+ for j := range chars {
+ i := rune(j)
+ web := inCategory(i)
+ pkg := unicode.Is(table, i)
+ if web != pkg {
+ fmt.Fprintf(os.Stderr, "%s: %U: web=%t pkg=%t\n", name, i, web, pkg)
+ count++
+ if count > 10 {
+ break
+ }
+ }
+ }
+}
+
+func parseScript(line string, scripts map[string][]Script) {
+ comment := strings.Index(line, "#")
+ if comment >= 0 {
+ line = line[0:comment]
+ }
+ line = strings.TrimSpace(line)
+ if len(line) == 0 {
+ return
+ }
+ field := strings.Split(line, ";")
+ if len(field) != 2 {
+ logger.Fatalf("%s: %d fields (expected 2)\n", line, len(field))
+ }
+ matches := scriptRe.FindStringSubmatch(line)
+ if len(matches) != 4 {
+ logger.Fatalf("%s: %d matches (expected 3)\n", line, len(matches))
+ }
+ lo, err := strconv.ParseUint(matches[1], 16, 64)
+ if err != nil {
+ logger.Fatalf("%.5s...: %s", line, err)
+ }
+ hi := lo
+ if len(matches[2]) > 2 { // ignore leading ..
+ hi, err = strconv.ParseUint(matches[2][2:], 16, 64)
+ if err != nil {
+ logger.Fatalf("%.5s...: %s", line, err)
+ }
+ }
+ name := matches[3]
+ scripts[name] = append(scripts[name], Script{uint32(lo), uint32(hi), name})
+}
+
+// The script tables have a lot of adjacent elements. Fold them together.
+func foldAdjacent(r []Script) []unicode.Range32 {
+ s := make([]unicode.Range32, 0, len(r))
+ j := 0
+ for i := 0; i < len(r); i++ {
+ if j > 0 && r[i].lo == s[j-1].Hi+1 {
+ s[j-1].Hi = r[i].hi
+ } else {
+ s = s[0 : j+1]
+ s[j] = unicode.Range32{
+ Lo: uint32(r[i].lo),
+ Hi: uint32(r[i].hi),
+ Stride: 1,
+ }
+ j++
+ }
+ }
+ return s
+}
+
+func fullScriptTest(list []string, installed map[string]*unicode.RangeTable, scripts map[string][]Script) {
+ for _, name := range list {
+ if _, ok := scripts[name]; !ok {
+ logger.Fatal("unknown script", name)
+ }
+ _, ok := installed[name]
+ if !ok {
+ logger.Fatal("unknown table", name)
+ }
+ for _, script := range scripts[name] {
+ for r := script.lo; r <= script.hi; r++ {
+ if !unicode.Is(installed[name], rune(r)) {
+ fmt.Fprintf(os.Stderr, "%U: not in script %s\n", r, name)
+ }
+ }
+ }
+ }
+}
+
+var deprecatedAliases = map[string]string{
+ "Sentence_Terminal": "STerm",
+}
+
+// PropList.txt has the same format as Scripts.txt so we can share its parser.
+func printScriptOrProperty(doProps bool) {
+ flag := "scripts"
+ flaglist := *scriptlist
+ file := "Scripts.txt"
+ table := scripts
+ installed := unicode.Scripts
+ if doProps {
+ flag = "props"
+ flaglist = *proplist
+ file = "PropList.txt"
+ table = props
+ installed = unicode.Properties
+ }
+ if flaglist == "" {
+ return
+ }
+ input := open(*url + file)
+ scanner := bufio.NewScanner(input)
+ for scanner.Scan() {
+ parseScript(scanner.Text(), table)
+ }
+ if scanner.Err() != nil {
+ logger.Fatal(scanner.Err())
+ }
+ input.close()
+
+ // Find out which scripts to dump
+ list := strings.Split(flaglist, ",")
+ if flaglist == "all" {
+ list = all(table)
+ }
+ if *test {
+ fullScriptTest(list, installed, table)
+ return
+ }
+
+ printf(
+ "// Generated by running\n"+
+ "// maketables --%s=%s --url=%s\n"+
+ "// DO NOT EDIT\n\n",
+ flag,
+ flaglist,
+ *url)
+ if flaglist == "all" {
+ if doProps {
+ println("// Properties is the set of Unicode property tables.")
+ println("var Properties = map[string] *RangeTable{")
+ } else {
+ println("// Scripts is the set of Unicode script tables.")
+ println("var Scripts = map[string] *RangeTable{")
+ }
+ for _, k := range all(table) {
+ printf("\t%q: %s,\n", k, k)
+ if alias, ok := deprecatedAliases[k]; ok {
+ printf("\t%q: %s,\n", alias, k)
+ }
+ }
+ print("}\n\n")
+ }
+
+ decl := make(sort.StringSlice, len(list)+len(deprecatedAliases))
+ ndecl := 0
+ for _, name := range list {
+ if doProps {
+ decl[ndecl] = fmt.Sprintf(
+ "\t%s = _%s;\t// %s is the set of Unicode characters with property %s.\n",
+ name, name, name, name)
+ } else {
+ decl[ndecl] = fmt.Sprintf(
+ "\t%s = _%s;\t// %s is the set of Unicode characters in script %s.\n",
+ name, name, name, name)
+ }
+ ndecl++
+ if alias, ok := deprecatedAliases[name]; ok {
+ decl[ndecl] = fmt.Sprintf(
+ "\t%[1]s = _%[2]s;\t// %[1]s is an alias for %[2]s.\n",
+ alias, name)
+ ndecl++
+ }
+ printf("var _%s = &RangeTable {\n", name)
+ ranges := foldAdjacent(table[name])
+ print("\tR16: []Range16{\n")
+ size := 16
+ count := &range16Count
+ for _, s := range ranges {
+ size, count = printRange(s.Lo, s.Hi, s.Stride, size, count)
+ }
+ print("\t},\n")
+ if off := findLatinOffset(ranges); off > 0 {
+ printf("\tLatinOffset: %d,\n", off)
+ }
+ print("}\n\n")
+ }
+ decl.Sort()
+ println("// These variables have type *RangeTable.")
+ println("var (")
+ for _, d := range decl {
+ print(d)
+ }
+ print(")\n\n")
+}
+
+func findLatinOffset(ranges []unicode.Range32) int {
+ i := 0
+ for i < len(ranges) && ranges[i].Hi <= unicode.MaxLatin1 {
+ i++
+ }
+ return i
+}
+
+const (
+ CaseUpper = 1 << iota
+ CaseLower
+ CaseTitle
+ CaseNone = 0 // must be zero
+ CaseMissing = -1 // character not present; not a valid case state
+)
+
+type caseState struct {
+ point rune
+ _case int
+ deltaToUpper rune
+ deltaToLower rune
+ deltaToTitle rune
+}
+
+// Is d a continuation of the state of c?
+func (c *caseState) adjacent(d *caseState) bool {
+ if d.point < c.point {
+ c, d = d, c
+ }
+ switch {
+ case d.point != c.point+1: // code points not adjacent (shouldn't happen)
+ return false
+ case d._case != c._case: // different cases
+ return c.upperLowerAdjacent(d)
+ case c._case == CaseNone:
+ return false
+ case c._case == CaseMissing:
+ return false
+ case d.deltaToUpper != c.deltaToUpper:
+ return false
+ case d.deltaToLower != c.deltaToLower:
+ return false
+ case d.deltaToTitle != c.deltaToTitle:
+ return false
+ }
+ return true
+}
+
+// Is d the same as c, but opposite in upper/lower case? this would make it
+// an element of an UpperLower sequence.
+func (c *caseState) upperLowerAdjacent(d *caseState) bool {
+ // check they're a matched case pair. we know they have adjacent values
+ switch {
+ case c._case == CaseUpper && d._case != CaseLower:
+ return false
+ case c._case == CaseLower && d._case != CaseUpper:
+ return false
+ }
+ // matched pair (at least in upper/lower). make the order Upper Lower
+ if c._case == CaseLower {
+ c, d = d, c
+ }
+ // for an Upper Lower sequence the deltas have to be in order
+ // c: 0 1 0
+ // d: -1 0 -1
+ switch {
+ case c.deltaToUpper != 0:
+ return false
+ case c.deltaToLower != 1:
+ return false
+ case c.deltaToTitle != 0:
+ return false
+ case d.deltaToUpper != -1:
+ return false
+ case d.deltaToLower != 0:
+ return false
+ case d.deltaToTitle != -1:
+ return false
+ }
+ return true
+}
+
+// Does this character start an UpperLower sequence?
+func (c *caseState) isUpperLower() bool {
+ // for an Upper Lower sequence the deltas have to be in order
+ // c: 0 1 0
+ switch {
+ case c.deltaToUpper != 0:
+ return false
+ case c.deltaToLower != 1:
+ return false
+ case c.deltaToTitle != 0:
+ return false
+ }
+ return true
+}
+
+// Does this character start a LowerUpper sequence?
+func (c *caseState) isLowerUpper() bool {
+ // for an Upper Lower sequence the deltas have to be in order
+ // c: -1 0 -1
+ switch {
+ case c.deltaToUpper != -1:
+ return false
+ case c.deltaToLower != 0:
+ return false
+ case c.deltaToTitle != -1:
+ return false
+ }
+ return true
+}
+
+func getCaseState(i rune) (c *caseState) {
+ c = &caseState{point: i, _case: CaseNone}
+ ch := &chars[i]
+ switch ch.codePoint {
+ case 0:
+ c._case = CaseMissing // Will get NUL wrong but that doesn't matter
+ return
+ case ch.upperCase:
+ c._case = CaseUpper
+ case ch.lowerCase:
+ c._case = CaseLower
+ case ch.titleCase:
+ c._case = CaseTitle
+ }
+ // Some things such as roman numeral U+2161 don't describe themselves
+ // as upper case, but have a lower case. Second-guess them.
+ if c._case == CaseNone && ch.lowerCase != 0 {
+ c._case = CaseUpper
+ }
+ // Same in the other direction.
+ if c._case == CaseNone && ch.upperCase != 0 {
+ c._case = CaseLower
+ }
+
+ if ch.upperCase != 0 {
+ c.deltaToUpper = ch.upperCase - i
+ }
+ if ch.lowerCase != 0 {
+ c.deltaToLower = ch.lowerCase - i
+ }
+ if ch.titleCase != 0 {
+ c.deltaToTitle = ch.titleCase - i
+ }
+ return
+}
+
+func printCases() {
+ if !*cases {
+ return
+ }
+ if *test {
+ fullCaseTest()
+ return
+ }
+ printf(
+ "// Generated by running\n"+
+ "// maketables --data=%s --casefolding=%s\n"+
+ "// DO NOT EDIT\n\n"+
+ "// CaseRanges is the table describing case mappings for all letters with\n"+
+ "// non-self mappings.\n"+
+ "var CaseRanges = _CaseRanges\n"+
+ "var _CaseRanges = []CaseRange {\n",
+ *dataURL, *casefoldingURL)
+
+ var startState *caseState // the start of a run; nil for not active
+ var prevState = &caseState{} // the state of the previous character
+ for i := range chars {
+ state := getCaseState(rune(i))
+ if state.adjacent(prevState) {
+ prevState = state
+ continue
+ }
+ // end of run (possibly)
+ printCaseRange(startState, prevState)
+ startState = nil
+ if state._case != CaseMissing && state._case != CaseNone {
+ startState = state
+ }
+ prevState = state
+ }
+ print("}\n")
+}
+
+func printCaseRange(lo, hi *caseState) {
+ if lo == nil {
+ return
+ }
+ if lo.deltaToUpper == 0 && lo.deltaToLower == 0 && lo.deltaToTitle == 0 {
+ // character represents itself in all cases - no need to mention it
+ return
+ }
+ switch {
+ case hi.point > lo.point && lo.isUpperLower():
+ printf("\t{0x%04X, 0x%04X, d{UpperLower, UpperLower, UpperLower}},\n",
+ lo.point, hi.point)
+ case hi.point > lo.point && lo.isLowerUpper():
+ logger.Fatalf("LowerUpper sequence: should not happen: %U. If it's real, need to fix To()", lo.point)
+ printf("\t{0x%04X, 0x%04X, d{LowerUpper, LowerUpper, LowerUpper}},\n",
+ lo.point, hi.point)
+ default:
+ printf("\t{0x%04X, 0x%04X, d{%d, %d, %d}},\n",
+ lo.point, hi.point,
+ lo.deltaToUpper, lo.deltaToLower, lo.deltaToTitle)
+ }
+}
+
+// If the cased value in the Char is 0, it means use the rune itself.
+func caseIt(r, cased rune) rune {
+ if cased == 0 {
+ return r
+ }
+ return cased
+}
+
+func fullCaseTest() {
+ for j, c := range chars {
+ i := rune(j)
+ lower := unicode.ToLower(i)
+ want := caseIt(i, c.lowerCase)
+ if lower != want {
+ fmt.Fprintf(os.Stderr, "lower %U should be %U is %U\n", i, want, lower)
+ }
+ upper := unicode.ToUpper(i)
+ want = caseIt(i, c.upperCase)
+ if upper != want {
+ fmt.Fprintf(os.Stderr, "upper %U should be %U is %U\n", i, want, upper)
+ }
+ title := unicode.ToTitle(i)
+ want = caseIt(i, c.titleCase)
+ if title != want {
+ fmt.Fprintf(os.Stderr, "title %U should be %U is %U\n", i, want, title)
+ }
+ }
+}
+
+func printLatinProperties() {
+ if *test {
+ return
+ }
+ println("var properties = [MaxLatin1+1]uint8{")
+ for code := 0; code <= unicode.MaxLatin1; code++ {
+ var property string
+ switch chars[code].category {
+ case "Cc", "": // NUL has no category.
+ property = "pC"
+ case "Cf": // soft hyphen, unique category, not printable.
+ property = "0"
+ case "Ll":
+ property = "pLl | pp"
+ case "Lo":
+ property = "pLo | pp"
+ case "Lu":
+ property = "pLu | pp"
+ case "Nd", "No":
+ property = "pN | pp"
+ case "Pc", "Pd", "Pe", "Pf", "Pi", "Po", "Ps":
+ property = "pP | pp"
+ case "Sc", "Sk", "Sm", "So":
+ property = "pS | pp"
+ case "Zs":
+ property = "pZ"
+ default:
+ logger.Fatalf("%U has unknown category %q", code, chars[code].category)
+ }
+ // Special case
+ if code == ' ' {
+ property = "pZ | pp"
+ }
+ printf("\t0x%02X: %s, // %q\n", code, property, code)
+ }
+ printf("}\n\n")
+}
+
+func printCasefold() {
+ // Build list of case-folding groups attached to each canonical folded char (typically lower case).
+ var caseOrbit = make([][]rune, MaxChar+1)
+ for j := range chars {
+ i := rune(j)
+ c := &chars[i]
+ if c.foldCase == 0 {
+ continue
+ }
+ orb := caseOrbit[c.foldCase]
+ if orb == nil {
+ orb = append(orb, c.foldCase)
+ }
+ caseOrbit[c.foldCase] = append(orb, i)
+ }
+
+ // Insert explicit 1-element groups when assuming [lower, upper] would be wrong.
+ for j := range chars {
+ i := rune(j)
+ c := &chars[i]
+ f := c.foldCase
+ if f == 0 {
+ f = i
+ }
+ orb := caseOrbit[f]
+ if orb == nil && (c.upperCase != 0 && c.upperCase != i || c.lowerCase != 0 && c.lowerCase != i) {
+ // Default assumption of [upper, lower] is wrong.
+ caseOrbit[i] = []rune{i}
+ }
+ }
+
+ // Delete the groups for which assuming [lower, upper] or [upper, lower] is right.
+ for i, orb := range caseOrbit {
+ if len(orb) == 2 && chars[orb[0]].upperCase == orb[1] && chars[orb[1]].lowerCase == orb[0] {
+ caseOrbit[i] = nil
+ }
+ if len(orb) == 2 && chars[orb[1]].upperCase == orb[0] && chars[orb[0]].lowerCase == orb[1] {
+ caseOrbit[i] = nil
+ }
+ }
+
+ // Record orbit information in chars.
+ for _, orb := range caseOrbit {
+ if orb == nil {
+ continue
+ }
+ sort.Slice(orb, func(i, j int) bool {
+ return orb[i] < orb[j]
+ })
+ c := orb[len(orb)-1]
+ for _, d := range orb {
+ chars[c].caseOrbit = d
+ c = d
+ }
+ }
+
+ printAsciiFold()
+ printCaseOrbit()
+
+ // Tables of category and script folding exceptions: code points
+ // that must be added when interpreting a particular category/script
+ // in a case-folding context.
+ cat := make(map[string]map[rune]bool)
+ for name := range category {
+ if x := foldExceptions(inCategory(name)); len(x) > 0 {
+ cat[name] = x
+ }
+ }
+
+ scr := make(map[string]map[rune]bool)
+ for name := range scripts {
+ if x := foldExceptions(inScript(name)); len(x) > 0 {
+ scr[name] = x
+ }
+ }
+
+ printCatFold("FoldCategory", cat)
+ printCatFold("FoldScript", scr)
+}
+
+// inCategory returns a list of all the runes in the category.
+func inCategory(name string) []rune {
+ var x []rune
+ for j := range chars {
+ i := rune(j)
+ c := &chars[i]
+ if c.category == name || len(name) == 1 && len(c.category) > 1 && c.category[0] == name[0] {
+ x = append(x, i)
+ }
+ }
+ return x
+}
+
+// inScript returns a list of all the runes in the script.
+func inScript(name string) []rune {
+ var x []rune
+ for _, s := range scripts[name] {
+ for c := s.lo; c <= s.hi; c++ {
+ x = append(x, rune(c))
+ }
+ }
+ return x
+}
+
+// foldExceptions returns a list of all the runes fold-equivalent
+// to runes in class but not in class themselves.
+func foldExceptions(class []rune) map[rune]bool {
+ // Create map containing class and all fold-equivalent chars.
+ m := make(map[rune]bool)
+ for _, r := range class {
+ c := &chars[r]
+ if c.caseOrbit == 0 {
+ // Just upper and lower.
+ if u := c.upperCase; u != 0 {
+ m[u] = true
+ }
+ if l := c.lowerCase; l != 0 {
+ m[l] = true
+ }
+ m[r] = true
+ continue
+ }
+ // Otherwise walk orbit.
+ r0 := r
+ for {
+ m[r] = true
+ r = chars[r].caseOrbit
+ if r == r0 {
+ break
+ }
+ }
+ }
+
+ // Remove class itself.
+ for _, r := range class {
+ delete(m, r)
+ }
+
+ // What's left is the exceptions.
+ return m
+}
+
+var comment = map[string]string{
+ "FoldCategory": "// FoldCategory maps a category name to a table of\n" +
+ "// code points outside the category that are equivalent under\n" +
+ "// simple case folding to code points inside the category.\n" +
+ "// If there is no entry for a category name, there are no such points.\n",
+
+ "FoldScript": "// FoldScript maps a script name to a table of\n" +
+ "// code points outside the script that are equivalent under\n" +
+ "// simple case folding to code points inside the script.\n" +
+ "// If there is no entry for a script name, there are no such points.\n",
+}
+
+func printAsciiFold() {
+ printf("var asciiFold = [MaxASCII + 1]uint16{\n")
+ for i := rune(0); i <= unicode.MaxASCII; i++ {
+ c := chars[i]
+ f := c.caseOrbit
+ if f == 0 {
+ if c.lowerCase != i && c.lowerCase != 0 {
+ f = c.lowerCase
+ } else if c.upperCase != i && c.upperCase != 0 {
+ f = c.upperCase
+ } else {
+ f = i
+ }
+ }
+ printf("\t0x%04X,\n", f)
+ }
+ printf("}\n\n")
+}
+
+func printCaseOrbit() {
+ if *test {
+ for j := range chars {
+ i := rune(j)
+ c := &chars[i]
+ f := c.caseOrbit
+ if f == 0 {
+ if c.lowerCase != i && c.lowerCase != 0 {
+ f = c.lowerCase
+ } else if c.upperCase != i && c.upperCase != 0 {
+ f = c.upperCase
+ } else {
+ f = i
+ }
+ }
+ if g := unicode.SimpleFold(i); g != f {
+ fmt.Fprintf(os.Stderr, "unicode.SimpleFold(%#U) = %#U, want %#U\n", i, g, f)
+ }
+ }
+ return
+ }
+
+ printf("var caseOrbit = []foldPair{\n")
+ for i := range chars {
+ c := &chars[i]
+ if c.caseOrbit != 0 {
+ printf("\t{0x%04X, 0x%04X},\n", i, c.caseOrbit)
+ foldPairCount++
+ }
+ }
+ printf("}\n\n")
+}
+
+func printCatFold(name string, m map[string]map[rune]bool) {
+ if *test {
+ var pkgMap map[string]*unicode.RangeTable
+ if name == "FoldCategory" {
+ pkgMap = unicode.FoldCategory
+ } else {
+ pkgMap = unicode.FoldScript
+ }
+ if len(pkgMap) != len(m) {
+ fmt.Fprintf(os.Stderr, "unicode.%s has %d elements, want %d\n", name, len(pkgMap), len(m))
+ return
+ }
+ for k, v := range m {
+ t, ok := pkgMap[k]
+ if !ok {
+ fmt.Fprintf(os.Stderr, "unicode.%s[%q] missing\n", name, k)
+ continue
+ }
+ n := 0
+ for _, r := range t.R16 {
+ for c := rune(r.Lo); c <= rune(r.Hi); c += rune(r.Stride) {
+ if !v[c] {
+ fmt.Fprintf(os.Stderr, "unicode.%s[%q] contains %#U, should not\n", name, k, c)
+ }
+ n++
+ }
+ }
+ for _, r := range t.R32 {
+ for c := rune(r.Lo); c <= rune(r.Hi); c += rune(r.Stride) {
+ if !v[c] {
+ fmt.Fprintf(os.Stderr, "unicode.%s[%q] contains %#U, should not\n", name, k, c)
+ }
+ n++
+ }
+ }
+ if n != len(v) {
+ fmt.Fprintf(os.Stderr, "unicode.%s[%q] has %d code points, want %d\n", name, k, n, len(v))
+ }
+ }
+ return
+ }
+
+ print(comment[name])
+ printf("var %s = map[string]*RangeTable{\n", name)
+ for _, name := range allCatFold(m) {
+ printf("\t%q: fold%s,\n", name, name)
+ }
+ printf("}\n\n")
+ for _, name := range allCatFold(m) {
+ class := m[name]
+ dumpRange(
+ fmt.Sprintf("var fold%s = &RangeTable{\n", name),
+ func(code rune) bool { return class[code] })
+ }
+}
+
+var range16Count = 0 // Number of entries in the 16-bit range tables.
+var range32Count = 0 // Number of entries in the 32-bit range tables.
+var foldPairCount = 0 // Number of fold pairs in the exception tables.
+
+func printSizes() {
+ if *test {
+ return
+ }
+ println()
+ printf("// Range entries: %d 16-bit, %d 32-bit, %d total.\n", range16Count, range32Count, range16Count+range32Count)
+ range16Bytes := range16Count * 3 * 2
+ range32Bytes := range32Count * 3 * 4
+ printf("// Range bytes: %d 16-bit, %d 32-bit, %d total.\n", range16Bytes, range32Bytes, range16Bytes+range32Bytes)
+ println()
+ printf("// Fold orbit bytes: %d pairs, %d bytes\n", foldPairCount, foldPairCount*2*2)
+}
