| // Copyright 2013 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| //go:build ignore |
| // +build ignore |
| |
| // Language tag table generator. |
| // Data read from the web. |
| |
| package main |
| |
| import ( |
| "bufio" |
| "flag" |
| "fmt" |
| "io" |
| "log" |
| "math" |
| "reflect" |
| "regexp" |
| "sort" |
| "strconv" |
| "strings" |
| |
| "golang.org/x/text/internal/gen" |
| "golang.org/x/text/internal/tag" |
| "golang.org/x/text/unicode/cldr" |
| ) |
| |
| var ( |
| test = flag.Bool("test", |
| false, |
| "test existing tables; can be used to compare web data with package data.") |
| outputFile = flag.String("output", |
| "tables.go", |
| "output file for generated tables") |
| ) |
| |
| var comment = []string{ |
| ` |
| lang holds an alphabetically sorted list of ISO-639 language identifiers. |
| All entries are 4 bytes. The index of the identifier (divided by 4) is the language tag. |
| For 2-byte language identifiers, the two successive bytes have the following meaning: |
| - if the first letter of the 2- and 3-letter ISO codes are the same: |
| the second and third letter of the 3-letter ISO code. |
| - otherwise: a 0 and a by 2 bits right-shifted index into altLangISO3. |
| For 3-byte language identifiers the 4th byte is 0.`, |
| ` |
| langNoIndex is a bit vector of all 3-letter language codes that are not used as an index |
| in lookup tables. The language ids for these language codes are derived directly |
| from the letters and are not consecutive.`, |
| ` |
| altLangISO3 holds an alphabetically sorted list of 3-letter language code alternatives |
| to 2-letter language codes that cannot be derived using the method described above. |
| Each 3-letter code is followed by its 1-byte langID.`, |
| ` |
| altLangIndex is used to convert indexes in altLangISO3 to langIDs.`, |
| ` |
| AliasMap maps langIDs to their suggested replacements.`, |
| ` |
| script is an alphabetically sorted list of ISO 15924 codes. The index |
| of the script in the string, divided by 4, is the internal scriptID.`, |
| ` |
| isoRegionOffset needs to be added to the index of regionISO to obtain the regionID |
| for 2-letter ISO codes. (The first isoRegionOffset regionIDs are reserved for |
| the UN.M49 codes used for groups.)`, |
| ` |
| regionISO holds a list of alphabetically sorted 2-letter ISO region codes. |
| Each 2-letter codes is followed by two bytes with the following meaning: |
| - [A-Z}{2}: the first letter of the 2-letter code plus these two |
| letters form the 3-letter ISO code. |
| - 0, n: index into altRegionISO3.`, |
| ` |
| regionTypes defines the status of a region for various standards.`, |
| ` |
| m49 maps regionIDs to UN.M49 codes. The first isoRegionOffset entries are |
| codes indicating collections of regions.`, |
| ` |
| m49Index gives indexes into fromM49 based on the three most significant bits |
| of a 10-bit UN.M49 code. To search an UN.M49 code in fromM49, search in |
| fromM49[m49Index[msb39(code)]:m49Index[msb3(code)+1]] |
| for an entry where the first 7 bits match the 7 lsb of the UN.M49 code. |
| The region code is stored in the 9 lsb of the indexed value.`, |
| ` |
| fromM49 contains entries to map UN.M49 codes to regions. See m49Index for details.`, |
| ` |
| altRegionISO3 holds a list of 3-letter region codes that cannot be |
| mapped to 2-letter codes using the default algorithm. This is a short list.`, |
| ` |
| altRegionIDs holds a list of regionIDs the positions of which match those |
| of the 3-letter ISO codes in altRegionISO3.`, |
| ` |
| variantNumSpecialized is the number of specialized variants in variants.`, |
| ` |
| suppressScript is an index from langID to the dominant script for that language, |
| if it exists. If a script is given, it should be suppressed from the language tag.`, |
| ` |
| likelyLang is a lookup table, indexed by langID, for the most likely |
| scripts and regions given incomplete information. If more entries exist for a |
| given language, region and script are the index and size respectively |
| of the list in likelyLangList.`, |
| ` |
| likelyLangList holds lists info associated with likelyLang.`, |
| ` |
| likelyRegion is a lookup table, indexed by regionID, for the most likely |
| languages and scripts given incomplete information. If more entries exist |
| for a given regionID, lang and script are the index and size respectively |
| of the list in likelyRegionList. |
| TODO: exclude containers and user-definable regions from the list.`, |
| ` |
| likelyRegionList holds lists info associated with likelyRegion.`, |
| ` |
| likelyScript is a lookup table, indexed by scriptID, for the most likely |
| languages and regions given a script.`, |
| ` |
| nRegionGroups is the number of region groups.`, |
| ` |
| regionInclusion maps region identifiers to sets of regions in regionInclusionBits, |
| where each set holds all groupings that are directly connected in a region |
| containment graph.`, |
| ` |
| regionInclusionBits is an array of bit vectors where every vector represents |
| a set of region groupings. These sets are used to compute the distance |
| between two regions for the purpose of language matching.`, |
| ` |
| regionInclusionNext marks, for each entry in regionInclusionBits, the set of |
| all groups that are reachable from the groups set in the respective entry.`, |
| } |
| |
| // TODO: consider changing some of these structures to tries. This can reduce |
| // memory, but may increase the need for memory allocations. This could be |
| // mitigated if we can piggyback on language tags for common cases. |
| |
| func failOnError(e error) { |
| if e != nil { |
| log.Panic(e) |
| } |
| } |
| |
| type setType int |
| |
| const ( |
| Indexed setType = 1 + iota // all elements must be of same size |
| Linear |
| ) |
| |
| type stringSet struct { |
| s []string |
| sorted, frozen bool |
| |
| // We often need to update values after the creation of an index is completed. |
| // We include a convenience map for keeping track of this. |
| update map[string]string |
| typ setType // used for checking. |
| } |
| |
| func (ss *stringSet) clone() stringSet { |
| c := *ss |
| c.s = append([]string(nil), c.s...) |
| return c |
| } |
| |
| func (ss *stringSet) setType(t setType) { |
| if ss.typ != t && ss.typ != 0 { |
| log.Panicf("type %d cannot be assigned as it was already %d", t, ss.typ) |
| } |
| } |
| |
| // parse parses a whitespace-separated string and initializes ss with its |
| // components. |
| func (ss *stringSet) parse(s string) { |
| scan := bufio.NewScanner(strings.NewReader(s)) |
| scan.Split(bufio.ScanWords) |
| for scan.Scan() { |
| ss.add(scan.Text()) |
| } |
| } |
| |
| func (ss *stringSet) assertChangeable() { |
| if ss.frozen { |
| log.Panic("attempt to modify a frozen stringSet") |
| } |
| } |
| |
| func (ss *stringSet) add(s string) { |
| ss.assertChangeable() |
| ss.s = append(ss.s, s) |
| ss.sorted = ss.frozen |
| } |
| |
| func (ss *stringSet) freeze() { |
| ss.compact() |
| ss.frozen = true |
| } |
| |
| func (ss *stringSet) compact() { |
| if ss.sorted { |
| return |
| } |
| a := ss.s |
| sort.Strings(a) |
| k := 0 |
| for i := 1; i < len(a); i++ { |
| if a[k] != a[i] { |
| a[k+1] = a[i] |
| k++ |
| } |
| } |
| ss.s = a[:k+1] |
| ss.sorted = ss.frozen |
| } |
| |
| type funcSorter struct { |
| fn func(a, b string) bool |
| sort.StringSlice |
| } |
| |
| func (s funcSorter) Less(i, j int) bool { |
| return s.fn(s.StringSlice[i], s.StringSlice[j]) |
| } |
| |
| func (ss *stringSet) sortFunc(f func(a, b string) bool) { |
| ss.compact() |
| sort.Sort(funcSorter{f, sort.StringSlice(ss.s)}) |
| } |
| |
| func (ss *stringSet) remove(s string) { |
| ss.assertChangeable() |
| if i, ok := ss.find(s); ok { |
| copy(ss.s[i:], ss.s[i+1:]) |
| ss.s = ss.s[:len(ss.s)-1] |
| } |
| } |
| |
| func (ss *stringSet) replace(ol, nu string) { |
| ss.s[ss.index(ol)] = nu |
| ss.sorted = ss.frozen |
| } |
| |
| func (ss *stringSet) index(s string) int { |
| ss.setType(Indexed) |
| i, ok := ss.find(s) |
| if !ok { |
| if i < len(ss.s) { |
| log.Panicf("find: item %q is not in list. Closest match is %q.", s, ss.s[i]) |
| } |
| log.Panicf("find: item %q is not in list", s) |
| |
| } |
| return i |
| } |
| |
| func (ss *stringSet) find(s string) (int, bool) { |
| ss.compact() |
| i := sort.SearchStrings(ss.s, s) |
| return i, i != len(ss.s) && ss.s[i] == s |
| } |
| |
| func (ss *stringSet) slice() []string { |
| ss.compact() |
| return ss.s |
| } |
| |
| func (ss *stringSet) updateLater(v, key string) { |
| if ss.update == nil { |
| ss.update = map[string]string{} |
| } |
| ss.update[v] = key |
| } |
| |
| // join joins the string and ensures that all entries are of the same length. |
| func (ss *stringSet) join() string { |
| ss.setType(Indexed) |
| n := len(ss.s[0]) |
| for _, s := range ss.s { |
| if len(s) != n { |
| log.Panicf("join: not all entries are of the same length: %q", s) |
| } |
| } |
| ss.s = append(ss.s, strings.Repeat("\xff", n)) |
| return strings.Join(ss.s, "") |
| } |
| |
| // ianaEntry holds information for an entry in the IANA Language Subtag Repository. |
| // All types use the same entry. |
| // See http://tools.ietf.org/html/bcp47#section-5.1 for a description of the various |
| // fields. |
| type ianaEntry struct { |
| typ string |
| description []string |
| scope string |
| added string |
| preferred string |
| deprecated string |
| suppressScript string |
| macro string |
| prefix []string |
| } |
| |
| type builder struct { |
| w *gen.CodeWriter |
| hw io.Writer // MultiWriter for w and w.Hash |
| data *cldr.CLDR |
| supp *cldr.SupplementalData |
| |
| // indices |
| locale stringSet // common locales |
| lang stringSet // canonical language ids (2 or 3 letter ISO codes) with data |
| langNoIndex stringSet // 3-letter ISO codes with no associated data |
| script stringSet // 4-letter ISO codes |
| region stringSet // 2-letter ISO or 3-digit UN M49 codes |
| variant stringSet // 4-8-alphanumeric variant code. |
| |
| // Region codes that are groups with their corresponding group IDs. |
| groups map[int]index |
| |
| // langInfo |
| registry map[string]*ianaEntry |
| } |
| |
| type index uint |
| |
| func newBuilder(w *gen.CodeWriter) *builder { |
| r := gen.OpenCLDRCoreZip() |
| defer r.Close() |
| d := &cldr.Decoder{} |
| data, err := d.DecodeZip(r) |
| failOnError(err) |
| b := builder{ |
| w: w, |
| hw: io.MultiWriter(w, w.Hash), |
| data: data, |
| supp: data.Supplemental(), |
| } |
| b.parseRegistry() |
| return &b |
| } |
| |
| func (b *builder) parseRegistry() { |
| r := gen.OpenIANAFile("assignments/language-subtag-registry") |
| defer r.Close() |
| b.registry = make(map[string]*ianaEntry) |
| |
| scan := bufio.NewScanner(r) |
| scan.Split(bufio.ScanWords) |
| var record *ianaEntry |
| for more := scan.Scan(); more; { |
| key := scan.Text() |
| more = scan.Scan() |
| value := scan.Text() |
| switch key { |
| case "Type:": |
| record = &ianaEntry{typ: value} |
| case "Subtag:", "Tag:": |
| if s := strings.SplitN(value, "..", 2); len(s) > 1 { |
| for a := s[0]; a <= s[1]; a = inc(a) { |
| b.addToRegistry(a, record) |
| } |
| } else { |
| b.addToRegistry(value, record) |
| } |
| case "Suppress-Script:": |
| record.suppressScript = value |
| case "Added:": |
| record.added = value |
| case "Deprecated:": |
| record.deprecated = value |
| case "Macrolanguage:": |
| record.macro = value |
| case "Preferred-Value:": |
| record.preferred = value |
| case "Prefix:": |
| record.prefix = append(record.prefix, value) |
| case "Scope:": |
| record.scope = value |
| case "Description:": |
| buf := []byte(value) |
| for more = scan.Scan(); more; more = scan.Scan() { |
| b := scan.Bytes() |
| if b[0] == '%' || b[len(b)-1] == ':' { |
| break |
| } |
| buf = append(buf, ' ') |
| buf = append(buf, b...) |
| } |
| record.description = append(record.description, string(buf)) |
| continue |
| default: |
| continue |
| } |
| more = scan.Scan() |
| } |
| if scan.Err() != nil { |
| log.Panic(scan.Err()) |
| } |
| } |
| |
| func (b *builder) addToRegistry(key string, entry *ianaEntry) { |
| if info, ok := b.registry[key]; ok { |
| if info.typ != "language" || entry.typ != "extlang" { |
| log.Fatalf("parseRegistry: tag %q already exists", key) |
| } |
| } else { |
| b.registry[key] = entry |
| } |
| } |
| |
| var commentIndex = make(map[string]string) |
| |
| func init() { |
| for _, s := range comment { |
| key := strings.TrimSpace(strings.SplitN(s, " ", 2)[0]) |
| commentIndex[key] = s |
| } |
| } |
| |
| func (b *builder) comment(name string) { |
| if s := commentIndex[name]; len(s) > 0 { |
| b.w.WriteComment(s) |
| } else { |
| fmt.Fprintln(b.w) |
| } |
| } |
| |
| func (b *builder) pf(f string, x ...interface{}) { |
| fmt.Fprintf(b.hw, f, x...) |
| fmt.Fprint(b.hw, "\n") |
| } |
| |
| func (b *builder) p(x ...interface{}) { |
| fmt.Fprintln(b.hw, x...) |
| } |
| |
| func (b *builder) addSize(s int) { |
| b.w.Size += s |
| b.pf("// Size: %d bytes", s) |
| } |
| |
| func (b *builder) writeConst(name string, x interface{}) { |
| b.comment(name) |
| b.w.WriteConst(name, x) |
| } |
| |
| // writeConsts computes f(v) for all v in values and writes the results |
| // as constants named _v to a single constant block. |
| func (b *builder) writeConsts(f func(string) int, values ...string) { |
| b.pf("const (") |
| for _, v := range values { |
| b.pf("\t_%s = %v", v, f(v)) |
| } |
| b.pf(")") |
| } |
| |
| // writeType writes the type of the given value, which must be a struct. |
| func (b *builder) writeType(value interface{}) { |
| b.comment(reflect.TypeOf(value).Name()) |
| b.w.WriteType(value) |
| } |
| |
| func (b *builder) writeSlice(name string, ss interface{}) { |
| b.writeSliceAddSize(name, 0, ss) |
| } |
| |
| func (b *builder) writeSliceAddSize(name string, extraSize int, ss interface{}) { |
| b.comment(name) |
| b.w.Size += extraSize |
| v := reflect.ValueOf(ss) |
| t := v.Type().Elem() |
| b.pf("// Size: %d bytes, %d elements", v.Len()*int(t.Size())+extraSize, v.Len()) |
| |
| fmt.Fprintf(b.w, "var %s = ", name) |
| b.w.WriteArray(ss) |
| b.p() |
| } |
| |
| type FromTo struct { |
| From, To uint16 |
| } |
| |
| func (b *builder) writeSortedMap(name string, ss *stringSet, index func(s string) uint16) { |
| ss.sortFunc(func(a, b string) bool { |
| return index(a) < index(b) |
| }) |
| m := []FromTo{} |
| for _, s := range ss.s { |
| m = append(m, FromTo{index(s), index(ss.update[s])}) |
| } |
| b.writeSlice(name, m) |
| } |
| |
| const base = 'z' - 'a' + 1 |
| |
| func strToInt(s string) uint { |
| v := uint(0) |
| for i := 0; i < len(s); i++ { |
| v *= base |
| v += uint(s[i] - 'a') |
| } |
| return v |
| } |
| |
| // converts the given integer to the original ASCII string passed to strToInt. |
| // len(s) must match the number of characters obtained. |
| func intToStr(v uint, s []byte) { |
| for i := len(s) - 1; i >= 0; i-- { |
| s[i] = byte(v%base) + 'a' |
| v /= base |
| } |
| } |
| |
| func (b *builder) writeBitVector(name string, ss []string) { |
| vec := make([]uint8, int(math.Ceil(math.Pow(base, float64(len(ss[0])))/8))) |
| for _, s := range ss { |
| v := strToInt(s) |
| vec[v/8] |= 1 << (v % 8) |
| } |
| b.writeSlice(name, vec) |
| } |
| |
| // TODO: convert this type into a list or two-stage trie. |
| func (b *builder) writeMapFunc(name string, m map[string]string, f func(string) uint16) { |
| b.comment(name) |
| v := reflect.ValueOf(m) |
| sz := v.Len() * (2 + int(v.Type().Key().Size())) |
| for _, k := range m { |
| sz += len(k) |
| } |
| b.addSize(sz) |
| keys := []string{} |
| b.pf(`var %s = map[string]uint16{`, name) |
| for k := range m { |
| keys = append(keys, k) |
| } |
| sort.Strings(keys) |
| for _, k := range keys { |
| b.pf("\t%q: %v,", k, f(m[k])) |
| } |
| b.p("}") |
| } |
| |
| func (b *builder) writeMap(name string, m interface{}) { |
| b.comment(name) |
| v := reflect.ValueOf(m) |
| sz := v.Len() * (2 + int(v.Type().Key().Size()) + int(v.Type().Elem().Size())) |
| b.addSize(sz) |
| f := strings.FieldsFunc(fmt.Sprintf("%#v", m), func(r rune) bool { |
| return strings.IndexRune("{}, ", r) != -1 |
| }) |
| sort.Strings(f[1:]) |
| b.pf(`var %s = %s{`, name, f[0]) |
| for _, kv := range f[1:] { |
| b.pf("\t%s,", kv) |
| } |
| b.p("}") |
| } |
| |
| func (b *builder) langIndex(s string) uint16 { |
| if s == "und" { |
| return 0 |
| } |
| if i, ok := b.lang.find(s); ok { |
| return uint16(i) |
| } |
| return uint16(strToInt(s)) + uint16(len(b.lang.s)) |
| } |
| |
| // inc advances the string to its lexicographical successor. |
| func inc(s string) string { |
| const maxTagLength = 4 |
| var buf [maxTagLength]byte |
| intToStr(strToInt(strings.ToLower(s))+1, buf[:len(s)]) |
| for i := 0; i < len(s); i++ { |
| if s[i] <= 'Z' { |
| buf[i] -= 'a' - 'A' |
| } |
| } |
| return string(buf[:len(s)]) |
| } |
| |
| func (b *builder) parseIndices() { |
| meta := b.supp.Metadata |
| |
| for k, v := range b.registry { |
| var ss *stringSet |
| switch v.typ { |
| case "language": |
| if len(k) == 2 || v.suppressScript != "" || v.scope == "special" { |
| b.lang.add(k) |
| continue |
| } else { |
| ss = &b.langNoIndex |
| } |
| case "region": |
| ss = &b.region |
| case "script": |
| ss = &b.script |
| case "variant": |
| ss = &b.variant |
| default: |
| continue |
| } |
| ss.add(k) |
| } |
| // Include any language for which there is data. |
| for _, lang := range b.data.Locales() { |
| if x := b.data.RawLDML(lang); false || |
| x.LocaleDisplayNames != nil || |
| x.Characters != nil || |
| x.Delimiters != nil || |
| x.Measurement != nil || |
| x.Dates != nil || |
| x.Numbers != nil || |
| x.Units != nil || |
| x.ListPatterns != nil || |
| x.Collations != nil || |
| x.Segmentations != nil || |
| x.Rbnf != nil || |
| x.Annotations != nil || |
| x.Metadata != nil { |
| |
| from := strings.Split(lang, "_") |
| if lang := from[0]; lang != "root" { |
| b.lang.add(lang) |
| } |
| } |
| } |
| // Include locales for plural rules, which uses a different structure. |
| for _, plurals := range b.data.Supplemental().Plurals { |
| for _, rules := range plurals.PluralRules { |
| for _, lang := range strings.Split(rules.Locales, " ") { |
| if lang = strings.Split(lang, "_")[0]; lang != "root" { |
| b.lang.add(lang) |
| } |
| } |
| } |
| } |
| // Include languages in likely subtags. |
| for _, m := range b.supp.LikelySubtags.LikelySubtag { |
| from := strings.Split(m.From, "_") |
| b.lang.add(from[0]) |
| } |
| // Include ISO-639 alpha-3 bibliographic entries. |
| for _, a := range meta.Alias.LanguageAlias { |
| if a.Reason == "bibliographic" { |
| b.langNoIndex.add(a.Type) |
| } |
| } |
| // Include regions in territoryAlias (not all are in the IANA registry!) |
| for _, reg := range b.supp.Metadata.Alias.TerritoryAlias { |
| if len(reg.Type) == 2 { |
| b.region.add(reg.Type) |
| } |
| } |
| |
| for _, s := range b.lang.s { |
| if len(s) == 3 { |
| b.langNoIndex.remove(s) |
| } |
| } |
| b.writeConst("NumLanguages", len(b.lang.slice())+len(b.langNoIndex.slice())) |
| b.writeConst("NumScripts", len(b.script.slice())) |
| b.writeConst("NumRegions", len(b.region.slice())) |
| |
| // Add dummy codes at the start of each list to represent "unspecified". |
| b.lang.add("---") |
| b.script.add("----") |
| b.region.add("---") |
| |
| // common locales |
| b.locale.parse(meta.DefaultContent.Locales) |
| } |
| |
| // TODO: region inclusion data will probably not be use used in future matchers. |
| |
| func (b *builder) computeRegionGroups() { |
| b.groups = make(map[int]index) |
| |
| // Create group indices. |
| for i := 1; b.region.s[i][0] < 'A'; i++ { // Base M49 indices on regionID. |
| b.groups[i] = index(len(b.groups)) |
| } |
| for _, g := range b.supp.TerritoryContainment.Group { |
| // Skip UN and EURO zone as they are flattening the containment |
| // relationship. |
| if g.Type == "EZ" || g.Type == "UN" { |
| continue |
| } |
| group := b.region.index(g.Type) |
| if _, ok := b.groups[group]; !ok { |
| b.groups[group] = index(len(b.groups)) |
| } |
| } |
| if len(b.groups) > 64 { |
| log.Fatalf("only 64 groups supported, found %d", len(b.groups)) |
| } |
| b.writeConst("nRegionGroups", len(b.groups)) |
| } |
| |
| var langConsts = []string{ |
| "af", "am", "ar", "az", "bg", "bn", "ca", "cs", "da", "de", "el", "en", "es", |
| "et", "fa", "fi", "fil", "fr", "gu", "he", "hi", "hr", "hu", "hy", "id", "is", |
| "it", "ja", "ka", "kk", "km", "kn", "ko", "ky", "lo", "lt", "lv", "mk", "ml", |
| "mn", "mo", "mr", "ms", "mul", "my", "nb", "ne", "nl", "no", "pa", "pl", "pt", |
| "ro", "ru", "sh", "si", "sk", "sl", "sq", "sr", "sv", "sw", "ta", "te", "th", |
| "tl", "tn", "tr", "uk", "ur", "uz", "vi", "zh", "zu", |
| |
| // constants for grandfathered tags (if not already defined) |
| "jbo", "ami", "bnn", "hak", "tlh", "lb", "nv", "pwn", "tao", "tay", "tsu", |
| "nn", "sfb", "vgt", "sgg", "cmn", "nan", "hsn", |
| } |
| |
| // writeLanguage generates all tables needed for language canonicalization. |
| func (b *builder) writeLanguage() { |
| meta := b.supp.Metadata |
| |
| b.writeConst("nonCanonicalUnd", b.lang.index("und")) |
| b.writeConsts(func(s string) int { return int(b.langIndex(s)) }, langConsts...) |
| b.writeConst("langPrivateStart", b.langIndex("qaa")) |
| b.writeConst("langPrivateEnd", b.langIndex("qtz")) |
| |
| // Get language codes that need to be mapped (overlong 3-letter codes, |
| // deprecated 2-letter codes, legacy and grandfathered tags.) |
| langAliasMap := stringSet{} |
| aliasTypeMap := map[string]AliasType{} |
| |
| // altLangISO3 get the alternative ISO3 names that need to be mapped. |
| altLangISO3 := stringSet{} |
| // Add dummy start to avoid the use of index 0. |
| altLangISO3.add("---") |
| altLangISO3.updateLater("---", "aa") |
| |
| lang := b.lang.clone() |
| for _, a := range meta.Alias.LanguageAlias { |
| if a.Replacement == "" { |
| a.Replacement = "und" |
| } |
| // TODO: support mapping to tags |
| repl := strings.SplitN(a.Replacement, "_", 2)[0] |
| if a.Reason == "overlong" { |
| if len(a.Replacement) == 2 && len(a.Type) == 3 { |
| lang.updateLater(a.Replacement, a.Type) |
| } |
| } else if len(a.Type) <= 3 { |
| switch a.Reason { |
| case "macrolanguage": |
| aliasTypeMap[a.Type] = Macro |
| case "deprecated": |
| // handled elsewhere |
| continue |
| case "bibliographic", "legacy": |
| if a.Type == "no" { |
| continue |
| } |
| aliasTypeMap[a.Type] = Legacy |
| default: |
| log.Fatalf("new %s alias: %s", a.Reason, a.Type) |
| } |
| langAliasMap.add(a.Type) |
| langAliasMap.updateLater(a.Type, repl) |
| } |
| } |
| // Manually add the mapping of "nb" (Norwegian) to its macro language. |
| // This can be removed if CLDR adopts this change. |
| langAliasMap.add("nb") |
| langAliasMap.updateLater("nb", "no") |
| aliasTypeMap["nb"] = Macro |
| |
| for k, v := range b.registry { |
| // Also add deprecated values for 3-letter ISO codes, which CLDR omits. |
| if v.typ == "language" && v.deprecated != "" && v.preferred != "" { |
| langAliasMap.add(k) |
| langAliasMap.updateLater(k, v.preferred) |
| aliasTypeMap[k] = Deprecated |
| } |
| } |
| // Fix CLDR mappings. |
| lang.updateLater("tl", "tgl") |
| lang.updateLater("sh", "hbs") |
| lang.updateLater("mo", "mol") |
| lang.updateLater("no", "nor") |
| lang.updateLater("tw", "twi") |
| lang.updateLater("nb", "nob") |
| lang.updateLater("ak", "aka") |
| lang.updateLater("bh", "bih") |
| |
| // Ensure that each 2-letter code is matched with a 3-letter code. |
| for _, v := range lang.s[1:] { |
| s, ok := lang.update[v] |
| if !ok { |
| if s, ok = lang.update[langAliasMap.update[v]]; !ok { |
| continue |
| } |
| lang.update[v] = s |
| } |
| if v[0] != s[0] { |
| altLangISO3.add(s) |
| altLangISO3.updateLater(s, v) |
| } |
| } |
| |
| // Complete canonicalized language tags. |
| lang.freeze() |
| for i, v := range lang.s { |
| // We can avoid these manual entries by using the IANA registry directly. |
| // Seems easier to update the list manually, as changes are rare. |
| // The panic in this loop will trigger if we miss an entry. |
| add := "" |
| if s, ok := lang.update[v]; ok { |
| if s[0] == v[0] { |
| add = s[1:] |
| } else { |
| add = string([]byte{0, byte(altLangISO3.index(s))}) |
| } |
| } else if len(v) == 3 { |
| add = "\x00" |
| } else { |
| log.Panicf("no data for long form of %q", v) |
| } |
| lang.s[i] += add |
| } |
| b.writeConst("lang", tag.Index(lang.join())) |
| |
| b.writeConst("langNoIndexOffset", len(b.lang.s)) |
| |
| // space of all valid 3-letter language identifiers. |
| b.writeBitVector("langNoIndex", b.langNoIndex.slice()) |
| |
| altLangIndex := []uint16{} |
| for i, s := range altLangISO3.slice() { |
| altLangISO3.s[i] += string([]byte{byte(len(altLangIndex))}) |
| if i > 0 { |
| idx := b.lang.index(altLangISO3.update[s]) |
| altLangIndex = append(altLangIndex, uint16(idx)) |
| } |
| } |
| b.writeConst("altLangISO3", tag.Index(altLangISO3.join())) |
| b.writeSlice("altLangIndex", altLangIndex) |
| |
| b.writeSortedMap("AliasMap", &langAliasMap, b.langIndex) |
| types := make([]AliasType, len(langAliasMap.s)) |
| for i, s := range langAliasMap.s { |
| types[i] = aliasTypeMap[s] |
| } |
| b.writeSlice("AliasTypes", types) |
| } |
| |
| var scriptConsts = []string{ |
| "Latn", "Hani", "Hans", "Hant", "Qaaa", "Qaai", "Qabx", "Zinh", "Zyyy", |
| "Zzzz", |
| } |
| |
| func (b *builder) writeScript() { |
| b.writeConsts(b.script.index, scriptConsts...) |
| b.writeConst("script", tag.Index(b.script.join())) |
| |
| supp := make([]uint8, len(b.lang.slice())) |
| for i, v := range b.lang.slice()[1:] { |
| if sc := b.registry[v].suppressScript; sc != "" { |
| supp[i+1] = uint8(b.script.index(sc)) |
| } |
| } |
| b.writeSlice("suppressScript", supp) |
| |
| // There is only one deprecated script in CLDR. This value is hard-coded. |
| // We check here if the code must be updated. |
| for _, a := range b.supp.Metadata.Alias.ScriptAlias { |
| if a.Type != "Qaai" { |
| log.Panicf("unexpected deprecated stript %q", a.Type) |
| } |
| } |
| } |
| |
| func parseM49(s string) int16 { |
| if len(s) == 0 { |
| return 0 |
| } |
| v, err := strconv.ParseUint(s, 10, 10) |
| failOnError(err) |
| return int16(v) |
| } |
| |
| var regionConsts = []string{ |
| "001", "419", "BR", "CA", "ES", "GB", "MD", "PT", "UK", "US", |
| "ZZ", "XA", "XC", "XK", // Unofficial tag for Kosovo. |
| } |
| |
| func (b *builder) writeRegion() { |
| b.writeConsts(b.region.index, regionConsts...) |
| |
| isoOffset := b.region.index("AA") |
| m49map := make([]int16, len(b.region.slice())) |
| fromM49map := make(map[int16]int) |
| altRegionISO3 := "" |
| altRegionIDs := []uint16{} |
| |
| b.writeConst("isoRegionOffset", isoOffset) |
| |
| // 2-letter region lookup and mapping to numeric codes. |
| regionISO := b.region.clone() |
| regionISO.s = regionISO.s[isoOffset:] |
| regionISO.sorted = false |
| |
| regionTypes := make([]byte, len(b.region.s)) |
| |
| // Is the region valid BCP 47? |
| for s, e := range b.registry { |
| if len(s) == 2 && s == strings.ToUpper(s) { |
| i := b.region.index(s) |
| for _, d := range e.description { |
| if strings.Contains(d, "Private use") { |
| regionTypes[i] = iso3166UserAssigned |
| } |
| } |
| regionTypes[i] |= bcp47Region |
| } |
| } |
| |
| // Is the region a valid ccTLD? |
| r := gen.OpenIANAFile("domains/root/db") |
| defer r.Close() |
| |
| buf, err := io.ReadAll(r) |
| failOnError(err) |
| re := regexp.MustCompile(`"/domains/root/db/([a-z]{2}).html"`) |
| for _, m := range re.FindAllSubmatch(buf, -1) { |
| i := b.region.index(strings.ToUpper(string(m[1]))) |
| regionTypes[i] |= ccTLD |
| } |
| |
| b.writeSlice("regionTypes", regionTypes) |
| |
| iso3Set := make(map[string]int) |
| update := func(iso2, iso3 string) { |
| i := regionISO.index(iso2) |
| if j, ok := iso3Set[iso3]; !ok && iso3[0] == iso2[0] { |
| regionISO.s[i] += iso3[1:] |
| iso3Set[iso3] = -1 |
| } else { |
| if ok && j >= 0 { |
| regionISO.s[i] += string([]byte{0, byte(j)}) |
| } else { |
| iso3Set[iso3] = len(altRegionISO3) |
| regionISO.s[i] += string([]byte{0, byte(len(altRegionISO3))}) |
| altRegionISO3 += iso3 |
| altRegionIDs = append(altRegionIDs, uint16(isoOffset+i)) |
| } |
| } |
| } |
| for _, tc := range b.supp.CodeMappings.TerritoryCodes { |
| i := regionISO.index(tc.Type) + isoOffset |
| if d := m49map[i]; d != 0 { |
| log.Panicf("%s found as a duplicate UN.M49 code of %03d", tc.Numeric, d) |
| } |
| m49 := parseM49(tc.Numeric) |
| m49map[i] = m49 |
| if r := fromM49map[m49]; r == 0 { |
| fromM49map[m49] = i |
| } else if r != i { |
| dep := b.registry[regionISO.s[r-isoOffset]].deprecated |
| if t := b.registry[tc.Type]; t != nil && dep != "" && (t.deprecated == "" || t.deprecated > dep) { |
| fromM49map[m49] = i |
| } |
| } |
| } |
| for _, ta := range b.supp.Metadata.Alias.TerritoryAlias { |
| if len(ta.Type) == 3 && ta.Type[0] <= '9' && len(ta.Replacement) == 2 { |
| from := parseM49(ta.Type) |
| if r := fromM49map[from]; r == 0 { |
| fromM49map[from] = regionISO.index(ta.Replacement) + isoOffset |
| } |
| } |
| } |
| for _, tc := range b.supp.CodeMappings.TerritoryCodes { |
| if len(tc.Alpha3) == 3 { |
| update(tc.Type, tc.Alpha3) |
| } |
| } |
| // This entries are not included in territoryCodes. Mostly 3-letter variants |
| // of deleted codes and an entry for QU. |
| for _, m := range []struct{ iso2, iso3 string }{ |
| {"CT", "CTE"}, |
| {"DY", "DHY"}, |
| {"HV", "HVO"}, |
| {"JT", "JTN"}, |
| {"MI", "MID"}, |
| {"NH", "NHB"}, |
| {"NQ", "ATN"}, |
| {"PC", "PCI"}, |
| {"PU", "PUS"}, |
| {"PZ", "PCZ"}, |
| {"RH", "RHO"}, |
| {"VD", "VDR"}, |
| {"WK", "WAK"}, |
| // These three-letter codes are used for others as well. |
| {"FQ", "ATF"}, |
| } { |
| update(m.iso2, m.iso3) |
| } |
| for i, s := range regionISO.s { |
| if len(s) != 4 { |
| regionISO.s[i] = s + " " |
| } |
| } |
| b.writeConst("regionISO", tag.Index(regionISO.join())) |
| b.writeConst("altRegionISO3", altRegionISO3) |
| b.writeSlice("altRegionIDs", altRegionIDs) |
| |
| // Create list of deprecated regions. |
| // TODO: consider inserting SF -> FI. Not included by CLDR, but is the only |
| // Transitionally-reserved mapping not included. |
| regionOldMap := stringSet{} |
| // Include regions in territoryAlias (not all are in the IANA registry!) |
| for _, reg := range b.supp.Metadata.Alias.TerritoryAlias { |
| if len(reg.Type) == 2 && reg.Reason == "deprecated" && len(reg.Replacement) == 2 { |
| regionOldMap.add(reg.Type) |
| regionOldMap.updateLater(reg.Type, reg.Replacement) |
| i, _ := regionISO.find(reg.Type) |
| j, _ := regionISO.find(reg.Replacement) |
| if k := m49map[i+isoOffset]; k == 0 { |
| m49map[i+isoOffset] = m49map[j+isoOffset] |
| } |
| } |
| } |
| b.writeSortedMap("regionOldMap", ®ionOldMap, func(s string) uint16 { |
| return uint16(b.region.index(s)) |
| }) |
| // 3-digit region lookup, groupings. |
| for i := 1; i < isoOffset; i++ { |
| m := parseM49(b.region.s[i]) |
| m49map[i] = m |
| fromM49map[m] = i |
| } |
| b.writeSlice("m49", m49map) |
| |
| const ( |
| searchBits = 7 |
| regionBits = 9 |
| ) |
| if len(m49map) >= 1<<regionBits { |
| log.Fatalf("Maximum number of regions exceeded: %d > %d", len(m49map), 1<<regionBits) |
| } |
| m49Index := [9]int16{} |
| fromM49 := []uint16{} |
| m49 := []int{} |
| for k, _ := range fromM49map { |
| m49 = append(m49, int(k)) |
| } |
| sort.Ints(m49) |
| for _, k := range m49[1:] { |
| val := (k & (1<<searchBits - 1)) << regionBits |
| fromM49 = append(fromM49, uint16(val|fromM49map[int16(k)])) |
| m49Index[1:][k>>searchBits] = int16(len(fromM49)) |
| } |
| b.writeSlice("m49Index", m49Index) |
| b.writeSlice("fromM49", fromM49) |
| } |
| |
| const ( |
| // TODO: put these lists in regionTypes as user data? Could be used for |
| // various optimizations and refinements and could be exposed in the API. |
| iso3166Except = "AC CP DG EA EU FX IC SU TA UK" |
| iso3166Trans = "AN BU CS NT TP YU ZR" // SF is not in our set of Regions. |
| // DY and RH are actually not deleted, but indeterminately reserved. |
| iso3166DelCLDR = "CT DD DY FQ HV JT MI NH NQ PC PU PZ RH VD WK YD" |
| ) |
| |
| const ( |
| iso3166UserAssigned = 1 << iota |
| ccTLD |
| bcp47Region |
| ) |
| |
| func find(list []string, s string) int { |
| for i, t := range list { |
| if t == s { |
| return i |
| } |
| } |
| return -1 |
| } |
| |
| // writeVariant generates per-variant information and creates a map from variant |
| // name to index value. We assign index values such that sorting multiple |
| // variants by index value will result in the correct order. |
| // There are two types of variants: specialized and general. Specialized variants |
| // are only applicable to certain language or language-script pairs. Generalized |
| // variants apply to any language. Generalized variants always sort after |
| // specialized variants. We will therefore always assign a higher index value |
| // to a generalized variant than any other variant. Generalized variants are |
| // sorted alphabetically among themselves. |
| // Specialized variants may also sort after other specialized variants. Such |
| // variants will be ordered after any of the variants they may follow. |
| // We assume that if a variant x is followed by a variant y, then for any prefix |
| // p of x, p-x is a prefix of y. This allows us to order tags based on the |
| // maximum of the length of any of its prefixes. |
| // TODO: it is possible to define a set of Prefix values on variants such that |
| // a total order cannot be defined to the point that this algorithm breaks. |
| // In other words, we cannot guarantee the same order of variants for the |
| // future using the same algorithm or for non-compliant combinations of |
| // variants. For this reason, consider using simple alphabetic sorting |
| // of variants and ignore Prefix restrictions altogether. |
| func (b *builder) writeVariant() { |
| generalized := stringSet{} |
| specialized := stringSet{} |
| specializedExtend := stringSet{} |
| // Collate the variants by type and check assumptions. |
| for _, v := range b.variant.slice() { |
| e := b.registry[v] |
| if len(e.prefix) == 0 { |
| generalized.add(v) |
| continue |
| } |
| c := strings.Split(e.prefix[0], "-") |
| hasScriptOrRegion := false |
| if len(c) > 1 { |
| _, hasScriptOrRegion = b.script.find(c[1]) |
| if !hasScriptOrRegion { |
| _, hasScriptOrRegion = b.region.find(c[1]) |
| |
| } |
| } |
| if len(c) == 1 || len(c) == 2 && hasScriptOrRegion { |
| // Variant is preceded by a language. |
| specialized.add(v) |
| continue |
| } |
| // Variant is preceded by another variant. |
| specializedExtend.add(v) |
| prefix := c[0] + "-" |
| if hasScriptOrRegion { |
| prefix += c[1] |
| } |
| for _, p := range e.prefix { |
| // Verify that the prefix minus the last element is a prefix of the |
| // predecessor element. |
| i := strings.LastIndex(p, "-") |
| pred := b.registry[p[i+1:]] |
| if find(pred.prefix, p[:i]) < 0 { |
| log.Fatalf("prefix %q for variant %q not consistent with predecessor spec", p, v) |
| } |
| // The sorting used below does not work in the general case. It works |
| // if we assume that variants that may be followed by others only have |
| // prefixes of the same length. Verify this. |
| count := strings.Count(p[:i], "-") |
| for _, q := range pred.prefix { |
| if c := strings.Count(q, "-"); c != count { |
| log.Fatalf("variant %q preceding %q has a prefix %q of size %d; want %d", p[i+1:], v, q, c, count) |
| } |
| } |
| if !strings.HasPrefix(p, prefix) { |
| log.Fatalf("prefix %q of variant %q should start with %q", p, v, prefix) |
| } |
| } |
| } |
| |
| // Sort extended variants. |
| a := specializedExtend.s |
| less := func(v, w string) bool { |
| // Sort by the maximum number of elements. |
| maxCount := func(s string) (max int) { |
| for _, p := range b.registry[s].prefix { |
| if c := strings.Count(p, "-"); c > max { |
| max = c |
| } |
| } |
| return |
| } |
| if cv, cw := maxCount(v), maxCount(w); cv != cw { |
| return cv < cw |
| } |
| // Sort by name as tie breaker. |
| return v < w |
| } |
| sort.Sort(funcSorter{less, sort.StringSlice(a)}) |
| specializedExtend.frozen = true |
| |
| // Create index from variant name to index. |
| variantIndex := make(map[string]uint8) |
| add := func(s []string) { |
| for _, v := range s { |
| variantIndex[v] = uint8(len(variantIndex)) |
| } |
| } |
| add(specialized.slice()) |
| add(specializedExtend.s) |
| numSpecialized := len(variantIndex) |
| add(generalized.slice()) |
| if n := len(variantIndex); n > 255 { |
| log.Fatalf("maximum number of variants exceeded: was %d; want <= 255", n) |
| } |
| b.writeMap("variantIndex", variantIndex) |
| b.writeConst("variantNumSpecialized", numSpecialized) |
| } |
| |
| func (b *builder) writeLanguageInfo() { |
| } |
| |
| // writeLikelyData writes tables that are used both for finding parent relations and for |
| // language matching. Each entry contains additional bits to indicate the status of the |
| // data to know when it cannot be used for parent relations. |
| func (b *builder) writeLikelyData() { |
| const ( |
| isList = 1 << iota |
| scriptInFrom |
| regionInFrom |
| ) |
| type ( // generated types |
| likelyScriptRegion struct { |
| region uint16 |
| script uint16 |
| flags uint8 |
| } |
| likelyLangScript struct { |
| lang uint16 |
| script uint16 |
| flags uint8 |
| } |
| likelyLangRegion struct { |
| lang uint16 |
| region uint16 |
| } |
| // likelyTag is used for getting likely tags for group regions, where |
| // the likely region might be a region contained in the group. |
| likelyTag struct { |
| lang uint16 |
| region uint16 |
| script uint16 |
| } |
| ) |
| var ( // generated variables |
| likelyRegionGroup = make([]likelyTag, len(b.groups)) |
| likelyLang = make([]likelyScriptRegion, len(b.lang.s)) |
| likelyRegion = make([]likelyLangScript, len(b.region.s)) |
| likelyScript = make([]likelyLangRegion, len(b.script.s)) |
| likelyLangList = []likelyScriptRegion{} |
| likelyRegionList = []likelyLangScript{} |
| ) |
| type fromTo struct { |
| from, to []string |
| } |
| langToOther := map[int][]fromTo{} |
| regionToOther := map[int][]fromTo{} |
| for _, m := range b.supp.LikelySubtags.LikelySubtag { |
| from := strings.Split(m.From, "_") |
| to := strings.Split(m.To, "_") |
| if len(to) != 3 { |
| log.Fatalf("invalid number of subtags in %q: found %d, want 3", m.To, len(to)) |
| } |
| if len(from) > 3 { |
| log.Fatalf("invalid number of subtags: found %d, want 1-3", len(from)) |
| } |
| if from[0] != to[0] && from[0] != "und" { |
| log.Fatalf("unexpected language change in expansion: %s -> %s", from, to) |
| } |
| if len(from) == 3 { |
| if from[2] != to[2] { |
| log.Fatalf("unexpected region change in expansion: %s -> %s", from, to) |
| } |
| if from[0] != "und" { |
| log.Fatalf("unexpected fully specified from tag: %s -> %s", from, to) |
| } |
| } |
| if len(from) == 1 || from[0] != "und" { |
| id := 0 |
| if from[0] != "und" { |
| id = b.lang.index(from[0]) |
| } |
| langToOther[id] = append(langToOther[id], fromTo{from, to}) |
| } else if len(from) == 2 && len(from[1]) == 4 { |
| sid := b.script.index(from[1]) |
| likelyScript[sid].lang = uint16(b.langIndex(to[0])) |
| likelyScript[sid].region = uint16(b.region.index(to[2])) |
| } else { |
| r := b.region.index(from[len(from)-1]) |
| if id, ok := b.groups[r]; ok { |
| if from[0] != "und" { |
| log.Fatalf("region changed unexpectedly: %s -> %s", from, to) |
| } |
| likelyRegionGroup[id].lang = uint16(b.langIndex(to[0])) |
| likelyRegionGroup[id].script = uint16(b.script.index(to[1])) |
| likelyRegionGroup[id].region = uint16(b.region.index(to[2])) |
| } else { |
| regionToOther[r] = append(regionToOther[r], fromTo{from, to}) |
| } |
| } |
| } |
| b.writeType(likelyLangRegion{}) |
| b.writeSlice("likelyScript", likelyScript) |
| |
| for id := range b.lang.s { |
| list := langToOther[id] |
| if len(list) == 1 { |
| likelyLang[id].region = uint16(b.region.index(list[0].to[2])) |
| likelyLang[id].script = uint16(b.script.index(list[0].to[1])) |
| } else if len(list) > 1 { |
| likelyLang[id].flags = isList |
| likelyLang[id].region = uint16(len(likelyLangList)) |
| likelyLang[id].script = uint16(len(list)) |
| for _, x := range list { |
| flags := uint8(0) |
| if len(x.from) > 1 { |
| if x.from[1] == x.to[2] { |
| flags = regionInFrom |
| } else { |
| flags = scriptInFrom |
| } |
| } |
| likelyLangList = append(likelyLangList, likelyScriptRegion{ |
| region: uint16(b.region.index(x.to[2])), |
| script: uint16(b.script.index(x.to[1])), |
| flags: flags, |
| }) |
| } |
| } |
| } |
| // TODO: merge suppressScript data with this table. |
| b.writeType(likelyScriptRegion{}) |
| b.writeSlice("likelyLang", likelyLang) |
| b.writeSlice("likelyLangList", likelyLangList) |
| |
| for id := range b.region.s { |
| list := regionToOther[id] |
| if len(list) == 1 { |
| likelyRegion[id].lang = uint16(b.langIndex(list[0].to[0])) |
| likelyRegion[id].script = uint16(b.script.index(list[0].to[1])) |
| if len(list[0].from) > 2 { |
| likelyRegion[id].flags = scriptInFrom |
| } |
| } else if len(list) > 1 { |
| likelyRegion[id].flags = isList |
| likelyRegion[id].lang = uint16(len(likelyRegionList)) |
| likelyRegion[id].script = uint16(len(list)) |
| for i, x := range list { |
| if len(x.from) == 2 && i != 0 || i > 0 && len(x.from) != 3 { |
| log.Fatalf("unspecified script must be first in list: %v at %d", x.from, i) |
| } |
| x := likelyLangScript{ |
| lang: uint16(b.langIndex(x.to[0])), |
| script: uint16(b.script.index(x.to[1])), |
| } |
| if len(list[0].from) > 2 { |
| x.flags = scriptInFrom |
| } |
| likelyRegionList = append(likelyRegionList, x) |
| } |
| } |
| } |
| b.writeType(likelyLangScript{}) |
| b.writeSlice("likelyRegion", likelyRegion) |
| b.writeSlice("likelyRegionList", likelyRegionList) |
| |
| b.writeType(likelyTag{}) |
| b.writeSlice("likelyRegionGroup", likelyRegionGroup) |
| } |
| |
| func (b *builder) writeRegionInclusionData() { |
| var ( |
| // mm holds for each group the set of groups with a distance of 1. |
| mm = make(map[int][]index) |
| |
| // containment holds for each group the transitive closure of |
| // containment of other groups. |
| containment = make(map[index][]index) |
| ) |
| for _, g := range b.supp.TerritoryContainment.Group { |
| // Skip UN and EURO zone as they are flattening the containment |
| // relationship. |
| if g.Type == "EZ" || g.Type == "UN" { |
| continue |
| } |
| group := b.region.index(g.Type) |
| groupIdx := b.groups[group] |
| for _, mem := range strings.Split(g.Contains, " ") { |
| r := b.region.index(mem) |
| mm[r] = append(mm[r], groupIdx) |
| if g, ok := b.groups[r]; ok { |
| mm[group] = append(mm[group], g) |
| containment[groupIdx] = append(containment[groupIdx], g) |
| } |
| } |
| } |
| |
| regionContainment := make([]uint64, len(b.groups)) |
| for _, g := range b.groups { |
| l := containment[g] |
| |
| // Compute the transitive closure of containment. |
| for i := 0; i < len(l); i++ { |
| l = append(l, containment[l[i]]...) |
| } |
| |
| // Compute the bitmask. |
| regionContainment[g] = 1 << g |
| for _, v := range l { |
| regionContainment[g] |= 1 << v |
| } |
| } |
| b.writeSlice("regionContainment", regionContainment) |
| |
| regionInclusion := make([]uint8, len(b.region.s)) |
| bvs := make(map[uint64]index) |
| // Make the first bitvector positions correspond with the groups. |
| for r, i := range b.groups { |
| bv := uint64(1 << i) |
| for _, g := range mm[r] { |
| bv |= 1 << g |
| } |
| bvs[bv] = i |
| regionInclusion[r] = uint8(bvs[bv]) |
| } |
| for r := 1; r < len(b.region.s); r++ { |
| if _, ok := b.groups[r]; !ok { |
| bv := uint64(0) |
| for _, g := range mm[r] { |
| bv |= 1 << g |
| } |
| if bv == 0 { |
| // Pick the world for unspecified regions. |
| bv = 1 << b.groups[b.region.index("001")] |
| } |
| if _, ok := bvs[bv]; !ok { |
| bvs[bv] = index(len(bvs)) |
| } |
| regionInclusion[r] = uint8(bvs[bv]) |
| } |
| } |
| b.writeSlice("regionInclusion", regionInclusion) |
| regionInclusionBits := make([]uint64, len(bvs)) |
| for k, v := range bvs { |
| regionInclusionBits[v] = uint64(k) |
| } |
| // Add bit vectors for increasingly large distances until a fixed point is reached. |
| regionInclusionNext := []uint8{} |
| for i := 0; i < len(regionInclusionBits); i++ { |
| bits := regionInclusionBits[i] |
| next := bits |
| for i := uint(0); i < uint(len(b.groups)); i++ { |
| if bits&(1<<i) != 0 { |
| next |= regionInclusionBits[i] |
| } |
| } |
| if _, ok := bvs[next]; !ok { |
| bvs[next] = index(len(bvs)) |
| regionInclusionBits = append(regionInclusionBits, next) |
| } |
| regionInclusionNext = append(regionInclusionNext, uint8(bvs[next])) |
| } |
| b.writeSlice("regionInclusionBits", regionInclusionBits) |
| b.writeSlice("regionInclusionNext", regionInclusionNext) |
| } |
| |
| type parentRel struct { |
| lang uint16 |
| script uint16 |
| maxScript uint16 |
| toRegion uint16 |
| fromRegion []uint16 |
| } |
| |
| func (b *builder) writeParents() { |
| b.writeType(parentRel{}) |
| |
| parents := []parentRel{} |
| |
| // Construct parent overrides. |
| n := 0 |
| for _, p := range b.data.Supplemental().ParentLocales.ParentLocale { |
| // Skipping non-standard scripts to root is implemented using addTags. |
| if p.Parent == "root" { |
| continue |
| } |
| |
| sub := strings.Split(p.Parent, "_") |
| parent := parentRel{lang: b.langIndex(sub[0])} |
| if len(sub) == 2 { |
| // TODO: check that all undefined scripts are indeed Latn in these |
| // cases. |
| parent.maxScript = uint16(b.script.index("Latn")) |
| parent.toRegion = uint16(b.region.index(sub[1])) |
| } else { |
| parent.script = uint16(b.script.index(sub[1])) |
| parent.maxScript = parent.script |
| parent.toRegion = uint16(b.region.index(sub[2])) |
| } |
| for _, c := range strings.Split(p.Locales, " ") { |
| region := b.region.index(c[strings.LastIndex(c, "_")+1:]) |
| parent.fromRegion = append(parent.fromRegion, uint16(region)) |
| } |
| parents = append(parents, parent) |
| n += len(parent.fromRegion) |
| } |
| b.writeSliceAddSize("parents", n*2, parents) |
| } |
| |
| func main() { |
| gen.Init() |
| |
| gen.Repackage("gen_common.go", "common.go", "language") |
| |
| w := gen.NewCodeWriter() |
| defer w.WriteGoFile("tables.go", "language") |
| |
| fmt.Fprintln(w, `import "golang.org/x/text/internal/tag"`) |
| |
| b := newBuilder(w) |
| gen.WriteCLDRVersion(w) |
| |
| b.parseIndices() |
| b.writeType(FromTo{}) |
| b.writeLanguage() |
| b.writeScript() |
| b.writeRegion() |
| b.writeVariant() |
| // TODO: b.writeLocale() |
| b.computeRegionGroups() |
| b.writeLikelyData() |
| b.writeRegionInclusionData() |
| b.writeParents() |
| } |