internal/language/compact: copy of language
Copy language package to internal/language/compact
The purpose is to expose the compact ID functionality
to other packages in x/text, but without exporting
it outside of x/text.
The next steps will be to delete respective
functionality in the original and destination
package and expose the functionality in the
the internal package.
Change-Id: I35ff1a041069729f75bab743148f4cf3f16c2058
Reviewed-on: https://go-review.googlesource.com/96637
Run-TryBot: Marcel van Lohuizen <mpvl@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Ross Light <light@google.com>
diff --git a/internal/language/compact/compact.go b/internal/language/compact/compact.go
new file mode 100644
index 0000000..0106331
--- /dev/null
+++ b/internal/language/compact/compact.go
@@ -0,0 +1,45 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language // import "golang.org/x/text/internal/language/compact"
+
+import (
+ "sort"
+ "strings"
+
+ "golang.org/x/text/internal/language"
+)
+
+type compactID uint16
+
+func getCoreIndex(t language.Tag) (id compactID, ok bool) {
+ cci, ok := language.GetCompactCore(t)
+ if !ok {
+ return 0, false
+ }
+ i := sort.Search(len(coreTags), func(i int) bool {
+ return cci <= coreTags[i]
+ })
+ if i == len(coreTags) || coreTags[i] != cci {
+ return 0, false
+ }
+ return compactID(i), true
+}
+
+func (c compactID) tag() language.Tag {
+ if int(c) >= len(coreTags) {
+ return specialTags[int(c)-len(coreTags)]
+ }
+ return coreTags[c].Tag()
+}
+
+var specialTags []language.Tag
+
+func init() {
+ tags := strings.Split(specialTagsStr, " ")
+ specialTags = make([]language.Tag, len(tags))
+ for i, t := range tags {
+ specialTags[i] = language.MustParse(t)
+ }
+}
diff --git a/internal/language/compact/coverage.go b/internal/language/compact/coverage.go
new file mode 100644
index 0000000..fdb6156
--- /dev/null
+++ b/internal/language/compact/coverage.go
@@ -0,0 +1,187 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+import (
+ "fmt"
+ "sort"
+
+ "golang.org/x/text/internal/language"
+)
+
+// The Coverage interface is used to define the level of coverage of an
+// internationalization service. Note that not all types are supported by all
+// services. As lists may be generated on the fly, it is recommended that users
+// of a Coverage cache the results.
+type Coverage interface {
+ // Tags returns the list of supported tags.
+ Tags() []Tag
+
+ // BaseLanguages returns the list of supported base languages.
+ BaseLanguages() []Base
+
+ // Scripts returns the list of supported scripts.
+ Scripts() []Script
+
+ // Regions returns the list of supported regions.
+ Regions() []Region
+}
+
+var (
+ // Supported defines a Coverage that lists all supported subtags. Tags
+ // always returns nil.
+ Supported Coverage = allSubtags{}
+)
+
+// TODO:
+// - Support Variants, numbering systems.
+// - CLDR coverage levels.
+// - Set of common tags defined in this package.
+
+type allSubtags struct{}
+
+// Regions returns the list of supported regions. As all regions are in a
+// consecutive range, it simply returns a slice of numbers in increasing order.
+// The "undefined" region is not returned.
+func (s allSubtags) Regions() []Region {
+ reg := make([]Region, language.NumRegions)
+ for i := range reg {
+ reg[i] = Region{language.Region(i + 1)}
+ }
+ return reg
+}
+
+// Scripts returns the list of supported scripts. As all scripts are in a
+// consecutive range, it simply returns a slice of numbers in increasing order.
+// The "undefined" script is not returned.
+func (s allSubtags) Scripts() []Script {
+ scr := make([]Script, language.NumScripts)
+ for i := range scr {
+ scr[i] = Script{language.Script(i + 1)}
+ }
+ return scr
+}
+
+// BaseLanguages returns the list of all supported base languages. It generates
+// the list by traversing the internal structures.
+func (s allSubtags) BaseLanguages() []Base {
+ bs := language.BaseLanguages()
+ base := make([]Base, len(bs))
+ for i, b := range bs {
+ base[i] = Base{b}
+ }
+ return base
+}
+
+// Tags always returns nil.
+func (s allSubtags) Tags() []Tag {
+ return nil
+}
+
+// coverage is used used by NewCoverage which is used as a convenient way for
+// creating Coverage implementations for partially defined data. Very often a
+// package will only need to define a subset of slices. coverage provides a
+// convenient way to do this. Moreover, packages using NewCoverage, instead of
+// their own implementation, will not break if later new slice types are added.
+type coverage struct {
+ tags func() []Tag
+ bases func() []Base
+ scripts func() []Script
+ regions func() []Region
+}
+
+func (s *coverage) Tags() []Tag {
+ if s.tags == nil {
+ return nil
+ }
+ return s.tags()
+}
+
+// bases implements sort.Interface and is used to sort base languages.
+type bases []Base
+
+func (b bases) Len() int {
+ return len(b)
+}
+
+func (b bases) Swap(i, j int) {
+ b[i], b[j] = b[j], b[i]
+}
+
+func (b bases) Less(i, j int) bool {
+ return b[i].langID < b[j].langID
+}
+
+// BaseLanguages returns the result from calling s.bases if it is specified or
+// otherwise derives the set of supported base languages from tags.
+func (s *coverage) BaseLanguages() []Base {
+ if s.bases == nil {
+ tags := s.Tags()
+ if len(tags) == 0 {
+ return nil
+ }
+ a := make([]Base, len(tags))
+ for i, t := range tags {
+ a[i] = Base{language.Language(t.lang())}
+ }
+ sort.Sort(bases(a))
+ k := 0
+ for i := 1; i < len(a); i++ {
+ if a[k] != a[i] {
+ k++
+ a[k] = a[i]
+ }
+ }
+ return a[:k+1]
+ }
+ return s.bases()
+}
+
+func (s *coverage) Scripts() []Script {
+ if s.scripts == nil {
+ return nil
+ }
+ return s.scripts()
+}
+
+func (s *coverage) Regions() []Region {
+ if s.regions == nil {
+ return nil
+ }
+ return s.regions()
+}
+
+// NewCoverage returns a Coverage for the given lists. It is typically used by
+// packages providing internationalization services to define their level of
+// coverage. A list may be of type []T or func() []T, where T is either Tag,
+// Base, Script or Region. The returned Coverage derives the value for Bases
+// from Tags if no func or slice for []Base is specified. For other unspecified
+// types the returned Coverage will return nil for the respective methods.
+func NewCoverage(list ...interface{}) Coverage {
+ s := &coverage{}
+ for _, x := range list {
+ switch v := x.(type) {
+ case func() []Base:
+ s.bases = v
+ case func() []Script:
+ s.scripts = v
+ case func() []Region:
+ s.regions = v
+ case func() []Tag:
+ s.tags = v
+ case []Base:
+ s.bases = func() []Base { return v }
+ case []Script:
+ s.scripts = func() []Script { return v }
+ case []Region:
+ s.regions = func() []Region { return v }
+ case []Tag:
+ s.tags = func() []Tag { return v }
+ default:
+ panic(fmt.Sprintf("language: unsupported set type %T", v))
+ }
+ }
+ return s
+}
diff --git a/internal/language/compact/coverage_test.go b/internal/language/compact/coverage_test.go
new file mode 100644
index 0000000..bbc092c
--- /dev/null
+++ b/internal/language/compact/coverage_test.go
@@ -0,0 +1,156 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+import (
+ "fmt"
+ "reflect"
+ "testing"
+
+ "golang.org/x/text/internal/language"
+)
+
+func TestSupported(t *testing.T) {
+ // To prove the results are correct for a type, we test that the number of
+ // results is identical to the number of results on record, that all results
+ // are distinct and that all results are valid.
+ tests := map[string]int{
+ "BaseLanguages": language.NumLanguages,
+ "Scripts": language.NumScripts,
+ "Regions": language.NumRegions,
+ "Tags": 0,
+ }
+ sup := reflect.ValueOf(Supported)
+ for name, num := range tests {
+ v := sup.MethodByName(name).Call(nil)[0]
+ if n := v.Len(); n != num {
+ t.Errorf("len(%s()) was %d; want %d", name, n, num)
+ }
+ dup := make(map[string]bool)
+ for i := 0; i < v.Len(); i++ {
+ x := v.Index(i).Interface()
+ // An invalid value will either cause a crash or result in a
+ // duplicate when passed to Sprint.
+ s := fmt.Sprint(x)
+ if dup[s] {
+ t.Errorf("%s: duplicate entry %q", name, s)
+ }
+ dup[s] = true
+ }
+ if len(dup) != v.Len() {
+ t.Errorf("%s: # unique entries was %d; want %d", name, len(dup), v.Len())
+ }
+ }
+}
+
+func TestNewCoverage(t *testing.T) {
+ bases := []Base{Base{0}, Base{3}, Base{7}}
+ scripts := []Script{Script{11}, Script{17}, Script{23}}
+ regions := []Region{Region{101}, Region{103}, Region{107}}
+ tags := []Tag{Make("pt"), Make("en"), Make("en-GB"), Make("en-US"), Make("pt-PT")}
+ fbases := func() []Base { return bases }
+ fscripts := func() []Script { return scripts }
+ fregions := func() []Region { return regions }
+ ftags := func() []Tag { return tags }
+
+ tests := []struct {
+ desc string
+ list []interface{}
+ bases []Base
+ scripts []Script
+ regions []Region
+ tags []Tag
+ }{
+ {
+ desc: "empty",
+ },
+ {
+ desc: "bases",
+ list: []interface{}{bases},
+ bases: bases,
+ },
+ {
+ desc: "scripts",
+ list: []interface{}{scripts},
+ scripts: scripts,
+ },
+ {
+ desc: "regions",
+ list: []interface{}{regions},
+ regions: regions,
+ },
+ {
+ desc: "bases derives from tags",
+ list: []interface{}{tags},
+ bases: []Base{Base{_en}, Base{_pt}},
+ tags: tags,
+ },
+ {
+ desc: "tags and bases",
+ list: []interface{}{tags, bases},
+ bases: bases,
+ tags: tags,
+ },
+ {
+ desc: "fully specified",
+ list: []interface{}{tags, bases, scripts, regions},
+ bases: bases,
+ scripts: scripts,
+ regions: regions,
+ tags: tags,
+ },
+ {
+ desc: "bases func",
+ list: []interface{}{fbases},
+ bases: bases,
+ },
+ {
+ desc: "scripts func",
+ list: []interface{}{fscripts},
+ scripts: scripts,
+ },
+ {
+ desc: "regions func",
+ list: []interface{}{fregions},
+ regions: regions,
+ },
+ {
+ desc: "tags func",
+ list: []interface{}{ftags},
+ bases: []Base{Base{_en}, Base{_pt}},
+ tags: tags,
+ },
+ {
+ desc: "tags and bases",
+ list: []interface{}{ftags, fbases},
+ bases: bases,
+ tags: tags,
+ },
+ {
+ desc: "fully specified",
+ list: []interface{}{ftags, fbases, fscripts, fregions},
+ bases: bases,
+ scripts: scripts,
+ regions: regions,
+ tags: tags,
+ },
+ }
+
+ for i, tt := range tests {
+ l := NewCoverage(tt.list...)
+ if a := l.BaseLanguages(); !reflect.DeepEqual(a, tt.bases) {
+ t.Errorf("%d:%s: BaseLanguages was %v; want %v", i, tt.desc, a, tt.bases)
+ }
+ if a := l.Scripts(); !reflect.DeepEqual(a, tt.scripts) {
+ t.Errorf("%d:%s: Scripts was %v; want %v", i, tt.desc, a, tt.scripts)
+ }
+ if a := l.Regions(); !reflect.DeepEqual(a, tt.regions) {
+ t.Errorf("%d:%s: Regions was %v; want %v", i, tt.desc, a, tt.regions)
+ }
+ if a := l.Tags(); !reflect.DeepEqual(a, tt.tags) {
+ t.Errorf("%d:%s: Tags was %v; want %v", i, tt.desc, a, tt.tags)
+ }
+ }
+}
diff --git a/internal/language/compact/examples_test.go b/internal/language/compact/examples_test.go
new file mode 100644
index 0000000..68caa3f
--- /dev/null
+++ b/internal/language/compact/examples_test.go
@@ -0,0 +1,413 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language_test
+
+import (
+ "fmt"
+ "net/http"
+
+ "golang.org/x/text/language"
+)
+
+func ExampleCanonType() {
+ p := func(id string) {
+ fmt.Printf("Default(%s) -> %s\n", id, language.Make(id))
+ fmt.Printf("BCP47(%s) -> %s\n", id, language.BCP47.Make(id))
+ fmt.Printf("Macro(%s) -> %s\n", id, language.Macro.Make(id))
+ fmt.Printf("All(%s) -> %s\n", id, language.All.Make(id))
+ }
+ p("en-Latn")
+ p("sh")
+ p("zh-cmn")
+ p("bjd")
+ p("iw-Latn-fonipa-u-cu-usd")
+ // Output:
+ // Default(en-Latn) -> en-Latn
+ // BCP47(en-Latn) -> en
+ // Macro(en-Latn) -> en-Latn
+ // All(en-Latn) -> en
+ // Default(sh) -> sr-Latn
+ // BCP47(sh) -> sh
+ // Macro(sh) -> sh
+ // All(sh) -> sr-Latn
+ // Default(zh-cmn) -> cmn
+ // BCP47(zh-cmn) -> cmn
+ // Macro(zh-cmn) -> zh
+ // All(zh-cmn) -> zh
+ // Default(bjd) -> drl
+ // BCP47(bjd) -> drl
+ // Macro(bjd) -> bjd
+ // All(bjd) -> drl
+ // Default(iw-Latn-fonipa-u-cu-usd) -> he-Latn-fonipa-u-cu-usd
+ // BCP47(iw-Latn-fonipa-u-cu-usd) -> he-Latn-fonipa-u-cu-usd
+ // Macro(iw-Latn-fonipa-u-cu-usd) -> iw-Latn-fonipa-u-cu-usd
+ // All(iw-Latn-fonipa-u-cu-usd) -> he-Latn-fonipa-u-cu-usd
+}
+
+func ExampleTag_Base() {
+ fmt.Println(language.Make("und").Base())
+ fmt.Println(language.Make("und-US").Base())
+ fmt.Println(language.Make("und-NL").Base())
+ fmt.Println(language.Make("und-419").Base()) // Latin America
+ fmt.Println(language.Make("und-ZZ").Base())
+ // Output:
+ // en Low
+ // en High
+ // nl High
+ // es Low
+ // en Low
+}
+
+func ExampleTag_Script() {
+ en := language.Make("en")
+ sr := language.Make("sr")
+ sr_Latn := language.Make("sr_Latn")
+ fmt.Println(en.Script())
+ fmt.Println(sr.Script())
+ // Was a script explicitly specified?
+ _, c := sr.Script()
+ fmt.Println(c == language.Exact)
+ _, c = sr_Latn.Script()
+ fmt.Println(c == language.Exact)
+ // Output:
+ // Latn High
+ // Cyrl Low
+ // false
+ // true
+}
+
+func ExampleTag_Region() {
+ ru := language.Make("ru")
+ en := language.Make("en")
+ fmt.Println(ru.Region())
+ fmt.Println(en.Region())
+ // Output:
+ // RU Low
+ // US Low
+}
+
+func ExampleRegion_TLD() {
+ us := language.MustParseRegion("US")
+ gb := language.MustParseRegion("GB")
+ uk := language.MustParseRegion("UK")
+ bu := language.MustParseRegion("BU")
+
+ fmt.Println(us.TLD())
+ fmt.Println(gb.TLD())
+ fmt.Println(uk.TLD())
+ fmt.Println(bu.TLD())
+
+ fmt.Println(us.Canonicalize().TLD())
+ fmt.Println(gb.Canonicalize().TLD())
+ fmt.Println(uk.Canonicalize().TLD())
+ fmt.Println(bu.Canonicalize().TLD())
+ // Output:
+ // US <nil>
+ // UK <nil>
+ // UK <nil>
+ // ZZ language: region is not a valid ccTLD
+ // US <nil>
+ // UK <nil>
+ // UK <nil>
+ // MM <nil>
+}
+
+func ExampleCompose() {
+ nl, _ := language.ParseBase("nl")
+ us, _ := language.ParseRegion("US")
+ de := language.Make("de-1901-u-co-phonebk")
+ jp := language.Make("ja-JP")
+ fi := language.Make("fi-x-ing")
+
+ u, _ := language.ParseExtension("u-nu-arabic")
+ x, _ := language.ParseExtension("x-piglatin")
+
+ // Combine a base language and region.
+ fmt.Println(language.Compose(nl, us))
+ // Combine a base language and extension.
+ fmt.Println(language.Compose(nl, x))
+ // Replace the region.
+ fmt.Println(language.Compose(jp, us))
+ // Combine several tags.
+ fmt.Println(language.Compose(us, nl, u))
+
+ // Replace the base language of a tag.
+ fmt.Println(language.Compose(de, nl))
+ fmt.Println(language.Compose(de, nl, u))
+ // Remove the base language.
+ fmt.Println(language.Compose(de, language.Base{}))
+ // Remove all variants.
+ fmt.Println(language.Compose(de, []language.Variant{}))
+ // Remove all extensions.
+ fmt.Println(language.Compose(de, []language.Extension{}))
+ fmt.Println(language.Compose(fi, []language.Extension{}))
+ // Remove all variants and extensions.
+ fmt.Println(language.Compose(de.Raw()))
+
+ // An error is gobbled or returned if non-nil.
+ fmt.Println(language.Compose(language.ParseRegion("ZA")))
+ fmt.Println(language.Compose(language.ParseRegion("HH")))
+
+ // Compose uses the same Default canonicalization as Make.
+ fmt.Println(language.Compose(language.Raw.Parse("en-Latn-UK")))
+
+ // Call compose on a different CanonType for different results.
+ fmt.Println(language.All.Compose(language.Raw.Parse("en-Latn-UK")))
+
+ // Output:
+ // nl-US <nil>
+ // nl-x-piglatin <nil>
+ // ja-US <nil>
+ // nl-US-u-nu-arabic <nil>
+ // nl-1901-u-co-phonebk <nil>
+ // nl-1901-u-co-phonebk-nu-arabic <nil>
+ // und-1901-u-co-phonebk <nil>
+ // de-u-co-phonebk <nil>
+ // de-1901 <nil>
+ // fi <nil>
+ // de <nil>
+ // und-ZA <nil>
+ // und language: subtag "HH" is well-formed but unknown
+ // en-Latn-GB <nil>
+ // en-GB <nil>
+}
+
+func ExampleParse_errors() {
+ for _, s := range []string{"Foo", "Bar", "Foobar"} {
+ _, err := language.Parse(s)
+ if err != nil {
+ if inv, ok := err.(language.ValueError); ok {
+ fmt.Println(inv.Subtag())
+ } else {
+ fmt.Println(s)
+ }
+ }
+ }
+ for _, s := range []string{"en", "aa-Uuuu", "AC", "ac-u"} {
+ _, err := language.Parse(s)
+ switch e := err.(type) {
+ case language.ValueError:
+ fmt.Printf("%s: culprit %q\n", s, e.Subtag())
+ case nil:
+ // No error.
+ default:
+ // A syntax error.
+ fmt.Printf("%s: ill-formed\n", s)
+ }
+ }
+ // Output:
+ // foo
+ // Foobar
+ // aa-Uuuu: culprit "Uuuu"
+ // AC: culprit "ac"
+ // ac-u: ill-formed
+}
+
+func ExampleParent() {
+ p := func(tag string) {
+ fmt.Printf("parent(%v): %v\n", tag, language.Make(tag).Parent())
+ }
+ p("zh-CN")
+
+ // Australian English inherits from World English.
+ p("en-AU")
+
+ // If the tag has a different maximized script from its parent, a tag with
+ // this maximized script is inserted. This allows different language tags
+ // which have the same base language and script in common to inherit from
+ // a common set of settings.
+ p("zh-HK")
+
+ // If the maximized script of the parent is not identical, CLDR will skip
+ // inheriting from it, as it means there will not be many entries in common
+ // and inheriting from it is nonsensical.
+ p("zh-Hant")
+
+ // The parent of a tag with variants and extensions is the tag with all
+ // variants and extensions removed.
+ p("de-1994-u-co-phonebk")
+
+ // Remove default script.
+ p("de-Latn-LU")
+
+ // Output:
+ // parent(zh-CN): zh
+ // parent(en-AU): en-001
+ // parent(zh-HK): zh-Hant
+ // parent(zh-Hant): und
+ // parent(de-1994-u-co-phonebk): de
+ // parent(de-Latn-LU): de
+}
+
+// ExampleMatcher_bestMatch gives some examples of getting the best match of
+// a set of tags to any of the tags of given set.
+func ExampleMatcher() {
+ // This is the set of tags from which we want to pick the best match. These
+ // can be, for example, the supported languages for some package.
+ tags := []language.Tag{
+ language.English,
+ language.BritishEnglish,
+ language.French,
+ language.Afrikaans,
+ language.BrazilianPortuguese,
+ language.EuropeanPortuguese,
+ language.Croatian,
+ language.SimplifiedChinese,
+ language.Raw.Make("iw-IL"),
+ language.Raw.Make("iw"),
+ language.Raw.Make("he"),
+ }
+ m := language.NewMatcher(tags)
+
+ // A simple match.
+ fmt.Println(m.Match(language.Make("fr")))
+
+ // Australian English is closer to British than American English.
+ fmt.Println(m.Match(language.Make("en-AU")))
+
+ // Default to the first tag passed to the Matcher if there is no match.
+ fmt.Println(m.Match(language.Make("ar")))
+
+ // Get the default tag.
+ fmt.Println(m.Match())
+
+ fmt.Println("----")
+
+ // Someone specifying sr-Latn is probably fine with getting Croatian.
+ fmt.Println(m.Match(language.Make("sr-Latn")))
+
+ // We match SimplifiedChinese, but with Low confidence.
+ fmt.Println(m.Match(language.TraditionalChinese))
+
+ // Serbian in Latin script is a closer match to Croatian than Traditional
+ // Chinese to Simplified Chinese.
+ fmt.Println(m.Match(language.TraditionalChinese, language.Make("sr-Latn")))
+
+ fmt.Println("----")
+
+ // In case a multiple variants of a language are available, the most spoken
+ // variant is typically returned.
+ fmt.Println(m.Match(language.Portuguese))
+
+ // Pick the first value passed to Match in case of a tie.
+ fmt.Println(m.Match(language.Dutch, language.Make("fr-BE"), language.Make("af-NA")))
+ fmt.Println(m.Match(language.Dutch, language.Make("af-NA"), language.Make("fr-BE")))
+
+ fmt.Println("----")
+
+ // If a Matcher is initialized with a language and it's deprecated version,
+ // it will distinguish between them.
+ fmt.Println(m.Match(language.Raw.Make("iw")))
+
+ // However, for non-exact matches, it will treat deprecated versions as
+ // equivalent and consider other factors first.
+ fmt.Println(m.Match(language.Raw.Make("he-IL")))
+
+ fmt.Println("----")
+
+ // User settings passed to the Unicode extension are ignored for matching
+ // and preserved in the returned tag.
+ fmt.Println(m.Match(language.Make("de-u-co-phonebk"), language.Make("fr-u-cu-frf")))
+
+ // Even if the matching language is different.
+ fmt.Println(m.Match(language.Make("de-u-co-phonebk"), language.Make("br-u-cu-frf")))
+
+ // If there is no matching language, the options of the first preferred tag are used.
+ fmt.Println(m.Match(language.Make("de-u-co-phonebk")))
+
+ // Output:
+ // fr 2 Exact
+ // en-GB 1 High
+ // en 0 No
+ // en 0 No
+ // ----
+ // hr 6 High
+ // zh-Hans 7 Low
+ // hr 6 High
+ // ----
+ // pt-BR 4 High
+ // fr 2 High
+ // af 3 High
+ // ----
+ // iw 9 Exact
+ // he 10 Exact
+ // ----
+ // fr-u-cu-frf 2 Exact
+ // fr-u-cu-frf 2 High
+ // en-u-co-phonebk 0 No
+
+ // TODO: "he" should be "he-u-rg-IL High"
+}
+
+func ExampleMatchStrings() {
+ // languages supported by this service:
+ matcher := language.NewMatcher([]language.Tag{
+ language.English, language.Dutch, language.German,
+ })
+
+ http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+ lang, _ := r.Cookie("lang")
+ tag, _ := language.MatchStrings(matcher, lang.String(), r.Header.Get("Accept-Language"))
+
+ fmt.Println("User language:", tag)
+ })
+}
+
+func ExampleComprehends() {
+ // Various levels of comprehensibility.
+ fmt.Println(language.Comprehends(language.English, language.English))
+ fmt.Println(language.Comprehends(language.AmericanEnglish, language.BritishEnglish))
+
+ // An explicit Und results in no match.
+ fmt.Println(language.Comprehends(language.English, language.Und))
+
+ fmt.Println("----")
+
+ // There is usually no mutual comprehensibility between different scripts.
+ fmt.Println(language.Comprehends(language.Make("en-Dsrt"), language.English))
+
+ // One exception is for Traditional versus Simplified Chinese, albeit with
+ // a low confidence.
+ fmt.Println(language.Comprehends(language.TraditionalChinese, language.SimplifiedChinese))
+
+ fmt.Println("----")
+
+ // A Swiss German speaker will often understand High German.
+ fmt.Println(language.Comprehends(language.Make("gsw"), language.Make("de")))
+
+ // The converse is not generally the case.
+ fmt.Println(language.Comprehends(language.Make("de"), language.Make("gsw")))
+
+ // Output:
+ // Exact
+ // High
+ // No
+ // ----
+ // No
+ // Low
+ // ----
+ // High
+ // No
+}
+
+func ExampleTag_values() {
+ us := language.MustParseRegion("US")
+ en := language.MustParseBase("en")
+
+ lang, _, region := language.AmericanEnglish.Raw()
+ fmt.Println(lang == en, region == us)
+
+ lang, _, region = language.BritishEnglish.Raw()
+ fmt.Println(lang == en, region == us)
+
+ // Tags can be compared for exact equivalence using '=='.
+ en_us, _ := language.Compose(en, us)
+ fmt.Println(en_us == language.AmericanEnglish)
+
+ // Output:
+ // true true
+ // true false
+ // true
+}
diff --git a/internal/language/compact/gen.go b/internal/language/compact/gen.go
new file mode 100644
index 0000000..5190040
--- /dev/null
+++ b/internal/language/compact/gen.go
@@ -0,0 +1,309 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Language tag table generator.
+// Data read from the web.
+
+package main
+
+import (
+ "flag"
+ "fmt"
+ "io"
+ "log"
+ "sort"
+ "strconv"
+ "strings"
+
+ "golang.org/x/text/internal/gen"
+ "golang.org/x/text/internal/language"
+ "golang.org/x/text/unicode/cldr"
+)
+
+var (
+ test = flag.Bool("test",
+ false,
+ "test existing tables; can be used to compare web data with package data.")
+ outputFile = flag.String("output",
+ "tables.go",
+ "output file for generated tables")
+)
+
+func main() {
+ gen.Init()
+
+ w := gen.NewCodeWriter()
+ defer w.WriteGoFile("tables.go", "language")
+
+ fmt.Fprintln(w, `import "golang.org/x/text/internal/language"`)
+
+ b := newBuilder(w)
+ gen.WriteCLDRVersion(w)
+
+ b.writeConstants()
+ b.writeCompactIndex()
+ b.writeMatchData()
+}
+
+type builder struct {
+ w *gen.CodeWriter
+ hw io.Writer // MultiWriter for w and w.Hash
+ data *cldr.CLDR
+ supp *cldr.SupplementalData
+}
+
+func (b *builder) langIndex(s string) uint16 {
+ return uint16(language.MustParseBase(s))
+}
+
+func (b *builder) regionIndex(s string) int {
+ return int(language.MustParseRegion(s))
+}
+
+func (b *builder) scriptIndex(s string) int {
+ return int(language.MustParseScript(s))
+}
+
+func newBuilder(w *gen.CodeWriter) *builder {
+ r := gen.OpenCLDRCoreZip()
+ defer r.Close()
+ d := &cldr.Decoder{}
+ data, err := d.DecodeZip(r)
+ if err != nil {
+ log.Fatal(err)
+ }
+ b := builder{
+ w: w,
+ hw: io.MultiWriter(w, w.Hash),
+ data: data,
+ supp: data.Supplemental(),
+ }
+ return &b
+}
+
+// writeConsts computes f(v) for all v in values and writes the results
+// as constants named _v to a single constant block.
+func (b *builder) writeConsts(f func(string) int, values ...string) {
+ fmt.Fprintln(b.w, "const (")
+ for _, v := range values {
+ fmt.Fprintf(b.w, "\t_%s = %v\n", v, f(v))
+ }
+ fmt.Fprintln(b.w, ")")
+}
+
+// TODO: region inclusion data will probably not be use used in future matchers.
+
+var langConsts = []string{
+ "de", "en", "fr", "it", "mo", "no", "nb", "pt", "sh", "mul", "und",
+}
+
+var scriptConsts = []string{
+ "Latn", "Hani", "Hans", "Hant", "Qaaa", "Qaai", "Qabx", "Zinh", "Zyyy",
+ "Zzzz",
+}
+
+var regionConsts = []string{
+ "001", "419", "BR", "CA", "ES", "GB", "MD", "PT", "UK", "US",
+ "ZZ", "XA", "XC", "XK", // Unofficial tag for Kosovo.
+}
+
+// writeLanguage generates all tables needed for language canonicalization.
+func (b *builder) writeConstants() {
+ b.writeConsts(func(s string) int { return int(b.langIndex(s)) }, langConsts...)
+ b.writeConsts(b.regionIndex, regionConsts...)
+ b.writeConsts(b.scriptIndex, scriptConsts...)
+}
+
+type mutualIntelligibility struct {
+ want, have uint16
+ distance uint8
+ oneway bool
+}
+
+type scriptIntelligibility struct {
+ wantLang, haveLang uint16
+ wantScript, haveScript uint8
+ distance uint8
+ // Always oneway
+}
+
+type regionIntelligibility struct {
+ lang uint16 // compact language id
+ script uint8 // 0 means any
+ group uint8 // 0 means any; if bit 7 is set it means inverse
+ distance uint8
+ // Always twoway.
+}
+
+// writeMatchData writes tables with languages and scripts for which there is
+// mutual intelligibility. The data is based on CLDR's languageMatching data.
+// Note that we use a different algorithm than the one defined by CLDR and that
+// we slightly modify the data. For example, we convert scores to confidence levels.
+// We also drop all region-related data as we use a different algorithm to
+// determine region equivalence.
+func (b *builder) writeMatchData() {
+ lm := b.supp.LanguageMatching.LanguageMatches
+ cldr.MakeSlice(&lm).SelectAnyOf("type", "written_new")
+
+ regionHierarchy := map[string][]string{}
+ for _, g := range b.supp.TerritoryContainment.Group {
+ regions := strings.Split(g.Contains, " ")
+ regionHierarchy[g.Type] = append(regionHierarchy[g.Type], regions...)
+ }
+ regionToGroups := make([]uint8, language.NumRegions)
+
+ idToIndex := map[string]uint8{}
+ for i, mv := range lm[0].MatchVariable {
+ if i > 6 {
+ log.Fatalf("Too many groups: %d", i)
+ }
+ idToIndex[mv.Id] = uint8(i + 1)
+ // TODO: also handle '-'
+ for _, r := range strings.Split(mv.Value, "+") {
+ todo := []string{r}
+ for k := 0; k < len(todo); k++ {
+ r := todo[k]
+ regionToGroups[b.regionIndex(r)] |= 1 << uint8(i)
+ todo = append(todo, regionHierarchy[r]...)
+ }
+ }
+ }
+ b.w.WriteVar("regionToGroups", regionToGroups)
+
+ // maps language id to in- and out-of-group region.
+ paradigmLocales := [][3]uint16{}
+ locales := strings.Split(lm[0].ParadigmLocales[0].Locales, " ")
+ for i := 0; i < len(locales); i += 2 {
+ x := [3]uint16{}
+ for j := 0; j < 2; j++ {
+ pc := strings.SplitN(locales[i+j], "-", 2)
+ x[0] = b.langIndex(pc[0])
+ if len(pc) == 2 {
+ x[1+j] = uint16(b.regionIndex(pc[1]))
+ }
+ }
+ paradigmLocales = append(paradigmLocales, x)
+ }
+ b.w.WriteVar("paradigmLocales", paradigmLocales)
+
+ b.w.WriteType(mutualIntelligibility{})
+ b.w.WriteType(scriptIntelligibility{})
+ b.w.WriteType(regionIntelligibility{})
+
+ matchLang := []mutualIntelligibility{}
+ matchScript := []scriptIntelligibility{}
+ matchRegion := []regionIntelligibility{}
+ // Convert the languageMatch entries in lists keyed by desired language.
+ for _, m := range lm[0].LanguageMatch {
+ // Different versions of CLDR use different separators.
+ desired := strings.Replace(m.Desired, "-", "_", -1)
+ supported := strings.Replace(m.Supported, "-", "_", -1)
+ d := strings.Split(desired, "_")
+ s := strings.Split(supported, "_")
+ if len(d) != len(s) {
+ log.Fatalf("not supported: desired=%q; supported=%q", desired, supported)
+ continue
+ }
+ distance, _ := strconv.ParseInt(m.Distance, 10, 8)
+ switch len(d) {
+ case 2:
+ if desired == supported && desired == "*_*" {
+ continue
+ }
+ // language-script pair.
+ matchScript = append(matchScript, scriptIntelligibility{
+ wantLang: uint16(b.langIndex(d[0])),
+ haveLang: uint16(b.langIndex(s[0])),
+ wantScript: uint8(b.scriptIndex(d[1])),
+ haveScript: uint8(b.scriptIndex(s[1])),
+ distance: uint8(distance),
+ })
+ if m.Oneway != "true" {
+ matchScript = append(matchScript, scriptIntelligibility{
+ wantLang: uint16(b.langIndex(s[0])),
+ haveLang: uint16(b.langIndex(d[0])),
+ wantScript: uint8(b.scriptIndex(s[1])),
+ haveScript: uint8(b.scriptIndex(d[1])),
+ distance: uint8(distance),
+ })
+ }
+ case 1:
+ if desired == supported && desired == "*" {
+ continue
+ }
+ if distance == 1 {
+ // nb == no is already handled by macro mapping. Check there
+ // really is only this case.
+ if d[0] != "no" || s[0] != "nb" {
+ log.Fatalf("unhandled equivalence %s == %s", s[0], d[0])
+ }
+ continue
+ }
+ // TODO: consider dropping oneway field and just doubling the entry.
+ matchLang = append(matchLang, mutualIntelligibility{
+ want: uint16(b.langIndex(d[0])),
+ have: uint16(b.langIndex(s[0])),
+ distance: uint8(distance),
+ oneway: m.Oneway == "true",
+ })
+ case 3:
+ if desired == supported && desired == "*_*_*" {
+ continue
+ }
+ if desired != supported {
+ // This is now supported by CLDR, but only one case, which
+ // should already be covered by paradigm locales. For instance,
+ // test case "und, en, en-GU, en-IN, en-GB ; en-ZA ; en-GB" in
+ // testdata/CLDRLocaleMatcherTest.txt tests this.
+ if supported != "en_*_GB" {
+ log.Fatalf("not supported: desired=%q; supported=%q", desired, supported)
+ }
+ continue
+ }
+ ri := regionIntelligibility{
+ lang: b.langIndex(d[0]),
+ distance: uint8(distance),
+ }
+ if d[1] != "*" {
+ ri.script = uint8(b.scriptIndex(d[1]))
+ }
+ switch {
+ case d[2] == "*":
+ ri.group = 0x80 // not contained in anything
+ case strings.HasPrefix(d[2], "$!"):
+ ri.group = 0x80
+ d[2] = "$" + d[2][len("$!"):]
+ fallthrough
+ case strings.HasPrefix(d[2], "$"):
+ ri.group |= idToIndex[d[2]]
+ }
+ matchRegion = append(matchRegion, ri)
+ default:
+ log.Fatalf("not supported: desired=%q; supported=%q", desired, supported)
+ }
+ }
+ sort.SliceStable(matchLang, func(i, j int) bool {
+ return matchLang[i].distance < matchLang[j].distance
+ })
+ b.w.WriteComment(`
+ matchLang holds pairs of langIDs of base languages that are typically
+ mutually intelligible. Each pair is associated with a confidence and
+ whether the intelligibility goes one or both ways.`)
+ b.w.WriteVar("matchLang", matchLang)
+
+ b.w.WriteComment(`
+ matchScript holds pairs of scriptIDs where readers of one script
+ can typically also read the other. Each is associated with a confidence.`)
+ sort.SliceStable(matchScript, func(i, j int) bool {
+ return matchScript[i].distance < matchScript[j].distance
+ })
+ b.w.WriteVar("matchScript", matchScript)
+
+ sort.SliceStable(matchRegion, func(i, j int) bool {
+ return matchRegion[i].distance < matchRegion[j].distance
+ })
+ b.w.WriteVar("matchRegion", matchRegion)
+}
diff --git a/internal/language/compact/gen_index.go b/internal/language/compact/gen_index.go
new file mode 100644
index 0000000..2a84a91
--- /dev/null
+++ b/internal/language/compact/gen_index.go
@@ -0,0 +1,108 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+// This file generates derivative tables based on the language package itself.
+
+import (
+ "fmt"
+ "log"
+ "sort"
+ "strings"
+
+ "golang.org/x/text/internal/language"
+)
+
+// Compact indices:
+// Note -va-X variants only apply to localization variants.
+// BCP variants only ever apply to language.
+// The only ambiguity between tags is with regions.
+
+func (b *builder) writeCompactIndex() {
+ // Collect all language tags for which we have any data in CLDR.
+ m := map[language.Tag]bool{}
+ for _, lang := range b.data.Locales() {
+ // We include all locales unconditionally to be consistent with en_US.
+ // We want en_US, even though it has no data associated with it.
+
+ // TODO: put any of the languages for which no data exists at the end
+ // of the index. This allows all components based on ICU to use that
+ // as the cutoff point.
+ // if x := data.RawLDML(lang); false ||
+ // x.LocaleDisplayNames != nil ||
+ // x.Characters != nil ||
+ // x.Delimiters != nil ||
+ // x.Measurement != nil ||
+ // x.Dates != nil ||
+ // x.Numbers != nil ||
+ // x.Units != nil ||
+ // x.ListPatterns != nil ||
+ // x.Collations != nil ||
+ // x.Segmentations != nil ||
+ // x.Rbnf != nil ||
+ // x.Annotations != nil ||
+ // x.Metadata != nil {
+
+ // TODO: support POSIX natively, albeit non-standard.
+ tag := language.Make(strings.Replace(lang, "_POSIX", "-u-va-posix", 1))
+ m[tag] = true
+ // }
+ }
+ // Include locales for plural rules, which uses a different structure.
+ for _, plurals := range b.supp.Plurals {
+ for _, rules := range plurals.PluralRules {
+ for _, lang := range strings.Split(rules.Locales, " ") {
+ m[language.Make(lang)] = true
+ }
+ }
+ }
+
+ var coreTags []language.CompactCoreInfo
+ var special []string
+
+ for t := range m {
+ if x := t.Extensions(); len(x) != 0 && fmt.Sprint(x) != "[u-va-posix]" {
+ log.Fatalf("Unexpected extension %v in %v", x, t)
+ }
+ if len(t.Variants()) == 0 && len(t.Extensions()) == 0 {
+ cci, ok := language.GetCompactCore(t)
+ if !ok {
+ log.Fatalf("Locale for non-basic language %q", t)
+ }
+ coreTags = append(coreTags, cci)
+ } else {
+ special = append(special, t.String())
+ }
+ }
+
+ w := b.w
+
+ sort.Slice(coreTags, func(i, j int) bool { return coreTags[i] < coreTags[j] })
+ sort.Strings(special)
+
+ w.WriteComment(`
+ NumCompactTags is the number of common tags. The maximum tag is
+ NumCompactTags-1.`)
+ w.WriteConst("NumCompactTags", len(m))
+
+ fmt.Fprintln(w, "const (")
+ for i, t := range coreTags {
+ fmt.Fprintf(w, "%s compactID = %d\n", ident(t.Tag().String()), i)
+ }
+ for i, t := range special {
+ fmt.Fprintf(w, "%s compactID = %d\n", ident(t), i+len(coreTags))
+ }
+ fmt.Fprintln(w, ")")
+
+ w.WriteVar("coreTags", coreTags)
+
+ w.WriteConst("specialTagsStr", strings.Join(special, " "))
+}
+
+func ident(s string) string {
+ return strings.Replace(s, "-", "", -1) + "Index"
+}
diff --git a/internal/language/compact/go1_1.go b/internal/language/compact/go1_1.go
new file mode 100644
index 0000000..380f4c0
--- /dev/null
+++ b/internal/language/compact/go1_1.go
@@ -0,0 +1,38 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !go1.2
+
+package language
+
+import "sort"
+
+func sortStable(s sort.Interface) {
+ ss := stableSort{
+ s: s,
+ pos: make([]int, s.Len()),
+ }
+ for i := range ss.pos {
+ ss.pos[i] = i
+ }
+ sort.Sort(&ss)
+}
+
+type stableSort struct {
+ s sort.Interface
+ pos []int
+}
+
+func (s *stableSort) Len() int {
+ return len(s.pos)
+}
+
+func (s *stableSort) Less(i, j int) bool {
+ return s.s.Less(i, j) || !s.s.Less(j, i) && s.pos[i] < s.pos[j]
+}
+
+func (s *stableSort) Swap(i, j int) {
+ s.s.Swap(i, j)
+ s.pos[i], s.pos[j] = s.pos[j], s.pos[i]
+}
diff --git a/internal/language/compact/go1_2.go b/internal/language/compact/go1_2.go
new file mode 100644
index 0000000..38268c5
--- /dev/null
+++ b/internal/language/compact/go1_2.go
@@ -0,0 +1,11 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.2
+
+package language
+
+import "sort"
+
+var sortStable = sort.Stable
diff --git a/internal/language/compact/httpexample_test.go b/internal/language/compact/httpexample_test.go
new file mode 100644
index 0000000..03c0ab9
--- /dev/null
+++ b/internal/language/compact/httpexample_test.go
@@ -0,0 +1,48 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language_test
+
+import (
+ "fmt"
+ "net/http"
+ "strings"
+
+ "golang.org/x/text/language"
+)
+
+// matcher is a language.Matcher configured for all supported languages.
+var matcher = language.NewMatcher([]language.Tag{
+ language.BritishEnglish,
+ language.Norwegian,
+ language.German,
+})
+
+// handler is a http.HandlerFunc.
+func handler(w http.ResponseWriter, r *http.Request) {
+ t, q, err := language.ParseAcceptLanguage(r.Header.Get("Accept-Language"))
+ // We ignore the error: the default language will be selected for t == nil.
+ tag, _, _ := matcher.Match(t...)
+ fmt.Printf("%17v (t: %6v; q: %3v; err: %v)\n", tag, t, q, err)
+}
+
+func ExampleParseAcceptLanguage() {
+ for _, al := range []string{
+ "nn;q=0.3, en-us;q=0.8, en,",
+ "gsw, en;q=0.7, en-US;q=0.8",
+ "gsw, nl, da",
+ "invalid",
+ } {
+ // Create dummy request with Accept-Language set and pass it to handler.
+ r, _ := http.NewRequest("GET", "example.com", strings.NewReader("Hello"))
+ r.Header.Set("Accept-Language", al)
+ handler(nil, r)
+ }
+
+ // Output:
+ // en-GB (t: [ en en-US nn]; q: [ 1 0.8 0.3]; err: <nil>)
+ // en-GB-u-rg-uszzzz (t: [ gsw en-US en]; q: [ 1 0.8 0.7]; err: <nil>)
+ // de (t: [ gsw nl da]; q: [ 1 1 1]; err: <nil>)
+ // en-GB (t: []; q: []; err: language: tag is not well-formed)
+}
diff --git a/internal/language/compact/language.go b/internal/language/compact/language.go
new file mode 100644
index 0000000..c4855b5
--- /dev/null
+++ b/internal/language/compact/language.go
@@ -0,0 +1,762 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:generate go run gen.go gen_index.go -output tables.go
+
+package language
+
+// TODO: Remove above NOTE after:
+// - verifying that tables are dropped correctly (most notably matcher tables).
+
+import (
+ "strings"
+
+ "golang.org/x/text/internal/language"
+)
+
+// Tag represents a BCP 47 language tag. It is used to specify an instance of a
+// specific language or locale. All language tag values are guaranteed to be
+// well-formed.
+type Tag struct {
+ language compactID
+ locale compactID
+ full fullTag // always a language.Tag for now.
+}
+
+type fullTag interface {
+ IsRoot() bool
+ Parent() language.Tag
+}
+
+func makeTag(t language.Tag) (tag Tag) {
+ if region := t.TypeForKey("rg"); len(region) == 6 && region[2:] == "zzzz" {
+ if r, err := language.ParseRegion(region[:2]); err == nil {
+ tFull := t
+ t, _ = t.SetTypeForKey("rg", "")
+ // TODO: should we not consider "va" for the language tag?
+ var exact1, exact2 bool
+ tag.language, exact1 = compactIndex(t)
+ t.RegionID = r
+ tag.locale, exact2 = compactIndex(t)
+ if !exact1 || !exact2 {
+ tag.full = tFull
+ }
+ return tag
+ }
+ }
+ lang, ok := compactIndex(t)
+ tag.language = lang
+ tag.locale = lang
+ if !ok {
+ tag.full = t
+ }
+ return tag
+}
+
+func (t *Tag) tag() language.Tag {
+ if t.full != nil {
+ return t.full.(language.Tag)
+ }
+ tag := t.language.tag()
+ if t.language != t.locale {
+ loc := t.locale.tag()
+ tag, _ = tag.SetTypeForKey("rg", strings.ToLower(loc.RegionID.String())+"zzzz")
+ }
+ return tag
+}
+
+func (t *Tag) mayHaveVariants() bool {
+ return t.full != nil || int(t.language) >= len(coreTags)
+}
+
+func (t *Tag) mayHaveExtensions() bool {
+ return t.full != nil ||
+ int(t.language) >= len(coreTags) ||
+ t.language != t.locale
+}
+
+// TODO: improve performance.
+func (t *Tag) lang() language.Language { return t.tag().LangID }
+func (t *Tag) region() language.Region { return t.tag().RegionID }
+func (t *Tag) script() language.Script { return t.tag().ScriptID }
+
+// Make is a convenience wrapper for Parse that omits the error.
+// In case of an error, a sensible default is returned.
+func Make(s string) Tag {
+ return Default.Make(s)
+}
+
+// Make is a convenience wrapper for c.Parse that omits the error.
+// In case of an error, a sensible default is returned.
+func (c CanonType) Make(s string) Tag {
+ t, _ := c.Parse(s)
+ return t
+}
+
+// Raw returns the raw base language, script and region, without making an
+// attempt to infer their values.
+func (t Tag) Raw() (b Base, s Script, r Region) {
+ tt := t.tag()
+ return Base{tt.LangID}, Script{tt.ScriptID}, Region{tt.RegionID}
+}
+
+// IsRoot returns true if t is equal to language "und".
+func (t Tag) IsRoot() bool {
+ if t.full != nil {
+ return t.full.IsRoot()
+ }
+ return t.language == _und
+}
+
+// CanonType can be used to enable or disable various types of canonicalization.
+type CanonType int
+
+const (
+ // Replace deprecated base languages with their preferred replacements.
+ DeprecatedBase CanonType = 1 << iota
+ // Replace deprecated scripts with their preferred replacements.
+ DeprecatedScript
+ // Replace deprecated regions with their preferred replacements.
+ DeprecatedRegion
+ // Remove redundant scripts.
+ SuppressScript
+ // Normalize legacy encodings. This includes legacy languages defined in
+ // CLDR as well as bibliographic codes defined in ISO-639.
+ Legacy
+ // Map the dominant language of a macro language group to the macro language
+ // subtag. For example cmn -> zh.
+ Macro
+ // The CLDR flag should be used if full compatibility with CLDR is required.
+ // There are a few cases where language.Tag may differ from CLDR. To follow all
+ // of CLDR's suggestions, use All|CLDR.
+ CLDR
+
+ // Raw can be used to Compose or Parse without Canonicalization.
+ Raw CanonType = 0
+
+ // Replace all deprecated tags with their preferred replacements.
+ Deprecated = DeprecatedBase | DeprecatedScript | DeprecatedRegion
+
+ // All canonicalizations recommended by BCP 47.
+ BCP47 = Deprecated | SuppressScript
+
+ // All canonicalizations.
+ All = BCP47 | Legacy | Macro
+
+ // Default is the canonicalization used by Parse, Make and Compose. To
+ // preserve as much information as possible, canonicalizations that remove
+ // potentially valuable information are not included. The Matcher is
+ // designed to recognize similar tags that would be the same if
+ // they were canonicalized using All.
+ Default = Deprecated | Legacy
+
+ canonLang = DeprecatedBase | Legacy | Macro
+
+ // TODO: LikelyScript, LikelyRegion: suppress similar to ICU.
+)
+
+// canonicalize returns the canonicalized equivalent of the tag and
+// whether there was any change.
+func canonicalize(c CanonType, t language.Tag) (language.Tag, bool) {
+ if c == Raw {
+ return t, false
+ }
+ changed := false
+ if c&SuppressScript != 0 {
+ if t.LangID.SuppressScript() == t.ScriptID {
+ t.ScriptID = 0
+ changed = true
+ }
+ }
+ if c&canonLang != 0 {
+ for {
+ if l, aliasType := t.LangID.Canonicalize(); l != t.LangID {
+ switch aliasType {
+ case language.Legacy:
+ if c&Legacy != 0 {
+ if t.LangID == _sh && t.ScriptID == 0 {
+ t.ScriptID = _Latn
+ }
+ t.LangID = l
+ changed = true
+ }
+ case language.Macro:
+ if c&Macro != 0 {
+ // We deviate here from CLDR. The mapping "nb" -> "no"
+ // qualifies as a typical Macro language mapping. However,
+ // for legacy reasons, CLDR maps "no", the macro language
+ // code for Norwegian, to the dominant variant "nb". This
+ // change is currently under consideration for CLDR as well.
+ // See http://unicode.org/cldr/trac/ticket/2698 and also
+ // http://unicode.org/cldr/trac/ticket/1790 for some of the
+ // practical implications. TODO: this check could be removed
+ // if CLDR adopts this change.
+ if c&CLDR == 0 || t.LangID != _nb {
+ changed = true
+ t.LangID = l
+ }
+ }
+ case language.Deprecated:
+ if c&DeprecatedBase != 0 {
+ if t.LangID == _mo && t.RegionID == 0 {
+ t.RegionID = _MD
+ }
+ t.LangID = l
+ changed = true
+ // Other canonicalization types may still apply.
+ continue
+ }
+ }
+ } else if c&Legacy != 0 && t.LangID == _no && c&CLDR != 0 {
+ t.LangID = _nb
+ changed = true
+ }
+ break
+ }
+ }
+ if c&DeprecatedScript != 0 {
+ if t.ScriptID == _Qaai {
+ changed = true
+ t.ScriptID = _Zinh
+ }
+ }
+ if c&DeprecatedRegion != 0 {
+ if r := t.RegionID.Canonicalize(); r != t.RegionID {
+ changed = true
+ t.RegionID = r
+ }
+ }
+ return t, changed
+}
+
+// Canonicalize returns the canonicalized equivalent of the tag.
+func (c CanonType) Canonicalize(t Tag) (Tag, error) {
+ // First try fast path.
+ if t.full == nil {
+ if _, changed := canonicalize(c, t.language.tag()); !changed {
+ return t, nil
+ }
+ }
+ // It is unlikely that one will canonicalize a tag after matching. So do
+ // a slow but simple approach here.
+ if tag, changed := canonicalize(c, t.tag()); changed {
+ tag.RemakeString()
+ return makeTag(tag), nil
+ }
+ return t, nil
+
+}
+
+// Confidence indicates the level of certainty for a given return value.
+// For example, Serbian may be written in Cyrillic or Latin script.
+// The confidence level indicates whether a value was explicitly specified,
+// whether it is typically the only possible value, or whether there is
+// an ambiguity.
+type Confidence int
+
+const (
+ No Confidence = iota // full confidence that there was no match
+ Low // most likely value picked out of a set of alternatives
+ High // value is generally assumed to be the correct match
+ Exact // exact match or explicitly specified value
+)
+
+var confName = []string{"No", "Low", "High", "Exact"}
+
+func (c Confidence) String() string {
+ return confName[c]
+}
+
+// String returns the canonical string representation of the language tag.
+func (t Tag) String() string {
+ return t.tag().String()
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (t Tag) MarshalText() (text []byte, err error) {
+ return t.tag().MarshalText()
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (t *Tag) UnmarshalText(text []byte) error {
+ var tag language.Tag
+ err := tag.UnmarshalText(text)
+ *t = makeTag(tag)
+ return err
+}
+
+// Base returns the base language of the language tag. If the base language is
+// unspecified, an attempt will be made to infer it from the context.
+// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
+func (t Tag) Base() (Base, Confidence) {
+ if b := t.lang(); b != 0 {
+ return Base{b}, Exact
+ }
+ tt := t.tag()
+ c := High
+ if tt.ScriptID == 0 && !tt.RegionID.IsCountry() {
+ c = Low
+ }
+ if tag, err := tt.Maximize(); err == nil && tag.LangID != 0 {
+ return Base{tag.LangID}, c
+ }
+ return Base{0}, No
+}
+
+// Script infers the script for the language tag. If it was not explicitly given, it will infer
+// a most likely candidate.
+// If more than one script is commonly used for a language, the most likely one
+// is returned with a low confidence indication. For example, it returns (Cyrl, Low)
+// for Serbian.
+// If a script cannot be inferred (Zzzz, No) is returned. We do not use Zyyy (undetermined)
+// as one would suspect from the IANA registry for BCP 47. In a Unicode context Zyyy marks
+// common characters (like 1, 2, 3, '.', etc.) and is therefore more like multiple scripts.
+// See http://www.unicode.org/reports/tr24/#Values for more details. Zzzz is also used for
+// unknown value in CLDR. (Zzzz, Exact) is returned if Zzzz was explicitly specified.
+// Note that an inferred script is never guaranteed to be the correct one. Latin is
+// almost exclusively used for Afrikaans, but Arabic has been used for some texts
+// in the past. Also, the script that is commonly used may change over time.
+// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
+func (t Tag) Script() (Script, Confidence) {
+ if scr := t.script(); scr != 0 {
+ return Script{scr}, Exact
+ }
+ tt := t.tag()
+ sc, c := language.Script(_Zzzz), No
+ if scr := tt.LangID.SuppressScript(); scr != 0 {
+ // Note: it is not always the case that a language with a suppress
+ // script value is only written in one script (e.g. kk, ms, pa).
+ if tt.RegionID == 0 {
+ return Script{scr}, High
+ }
+ sc, c = scr, High
+ }
+ if tag, err := tt.Maximize(); err == nil {
+ if tag.ScriptID != sc {
+ sc, c = tag.ScriptID, Low
+ }
+ } else {
+ tt, _ = canonicalize(Deprecated|Macro, tt)
+ if tag, err := tt.Maximize(); err == nil && tag.ScriptID != sc {
+ sc, c = tag.ScriptID, Low
+ }
+ }
+ return Script{sc}, c
+}
+
+// Region returns the region for the language tag. If it was not explicitly given, it will
+// infer a most likely candidate from the context.
+// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
+func (t Tag) Region() (Region, Confidence) {
+ if r := t.region(); r != 0 {
+ return Region{r}, Exact
+ }
+ tt := t.tag()
+ if tt, err := tt.Maximize(); err == nil {
+ return Region{tt.RegionID}, Low // TODO: differentiate between high and low.
+ }
+ tt, _ = canonicalize(Deprecated|Macro, tt)
+ if tag, err := tt.Maximize(); err == nil {
+ return Region{tag.RegionID}, Low
+ }
+ return Region{_ZZ}, No // TODO: return world instead of undetermined?
+}
+
+// Variants returns the variants specified explicitly for this language tag.
+// or nil if no variant was specified.
+func (t Tag) Variants() []Variant {
+ if !t.mayHaveVariants() {
+ return nil
+ }
+ v := []Variant{}
+ x, str := "", t.tag().Variants()
+ for str != "" {
+ x, str = nextToken(str)
+ v = append(v, Variant{x})
+ }
+ return v
+}
+
+// Parent returns the CLDR parent of t. In CLDR, missing fields in data for a
+// specific language are substituted with fields from the parent language.
+// The parent for a language may change for newer versions of CLDR.
+func (t Tag) Parent() Tag {
+ if t.full != nil {
+ return makeTag(t.full.Parent())
+ }
+ if t.language != t.locale {
+ // Simulate stripping -u-rg-xxxxxx
+ return Tag{language: t.language, locale: t.language}
+ }
+ // TODO: use parent lookup table once cycle from internal package is
+ // removed. Probably by internalizing the table and declaring this fast
+ // enough.
+ // lang := compactID(internal.Parent(uint16(t.language)))
+ lang, _ := compactIndex(t.language.tag().Parent())
+ return Tag{language: lang, locale: lang}
+}
+
+// returns token t and the rest of the string.
+func nextToken(s string) (t, tail string) {
+ p := strings.Index(s[1:], "-")
+ if p == -1 {
+ return s[1:], ""
+ }
+ p++
+ return s[1:p], s[p:]
+}
+
+// Extension is a single BCP 47 extension.
+type Extension struct {
+ s string
+}
+
+// String returns the string representation of the extension, including the
+// type tag.
+func (e Extension) String() string {
+ return e.s
+}
+
+// ParseExtension parses s as an extension and returns it on success.
+func ParseExtension(s string) (e Extension, err error) {
+ ext, err := language.ParseExtension(s)
+ return Extension{ext}, err
+}
+
+// Type returns the one-byte extension type of e. It returns 0 for the zero
+// exception.
+func (e Extension) Type() byte {
+ if e.s == "" {
+ return 0
+ }
+ return e.s[0]
+}
+
+// Tokens returns the list of tokens of e.
+func (e Extension) Tokens() []string {
+ return strings.Split(e.s, "-")
+}
+
+// Extension returns the extension of type x for tag t. It will return
+// false for ok if t does not have the requested extension. The returned
+// extension will be invalid in this case.
+func (t Tag) Extension(x byte) (ext Extension, ok bool) {
+ if !t.mayHaveExtensions() {
+ return Extension{}, false
+ }
+ e, ok := t.tag().Extension(x)
+ return Extension{e}, ok
+}
+
+// Extensions returns all extensions of t.
+func (t Tag) Extensions() []Extension {
+ if !t.mayHaveExtensions() {
+ return nil
+ }
+ e := []Extension{}
+ for _, ext := range t.tag().Extensions() {
+ e = append(e, Extension{ext})
+ }
+ return e
+}
+
+// TypeForKey returns the type associated with the given key, where key and type
+// are of the allowed values defined for the Unicode locale extension ('u') in
+// http://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
+// TypeForKey will traverse the inheritance chain to get the correct value.
+func (t Tag) TypeForKey(key string) string {
+ if !t.mayHaveExtensions() {
+ if key != "rg" && key != "va" {
+ return ""
+ }
+ }
+ return t.tag().TypeForKey(key)
+}
+
+// SetTypeForKey returns a new Tag with the key set to type, where key and type
+// are of the allowed values defined for the Unicode locale extension ('u') in
+// http://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
+// An empty value removes an existing pair with the same key.
+func (t Tag) SetTypeForKey(key, value string) (Tag, error) {
+ tt, err := t.tag().SetTypeForKey(key, value)
+ return makeTag(tt), err
+}
+
+// CompactIndex returns an index, where 0 <= index < NumCompactTags, for tags
+// for which data exists in the text repository.The index will change over time
+// and should not be stored in persistent storage. If t does not match a compact
+// index, exact will be false and the compact index will be returned for the
+// first match after repeatedly taking the Parent of t.
+func CompactIndex(t Tag) (index int, exact bool) {
+ return int(t.language), t.full == nil
+}
+
+// TODO: make these functions and methods public once we settle on the API and
+//
+
+// regionalCompactIndex returns the CompactIndex for the regional variant of this
+// tag. This index is used to indicate region-specific overrides, such as
+// default currency, default calendar and week data, default time cycle, and
+// default measurement system and unit preferences.
+//
+// For instance, the tag en-GB-u-rg-uszzzz specifies British English with US
+// settings for currency, number formatting, etc. The CompactIndex for this tag
+// will be that for en-GB, while the regionalCompactIndex will be the one
+// corresponding to en-US.
+func regionalCompactIndex(t Tag) (index int, exact bool) {
+ return int(t.locale), t.full == nil
+}
+
+// languageTag returns t stripped of regional variant indicators.
+//
+// At the moment this means it is stripped of a regional and variant subtag "rg"
+// and "va" in the "u" extension.
+func (t Tag) languageTag() Tag {
+ if t.full == nil {
+ return Tag{language: t.language, locale: t.language}
+ }
+ tt := t.tag()
+ tt.SetTypeForKey("rg", "")
+ tt.SetTypeForKey("va", "")
+ return makeTag(tt)
+}
+
+// regionalTag returns the regional variant of the tag.
+//
+// At the moment this means that the region is set from the regional subtag
+// "rg" in the "u" extension.
+func (t Tag) regionalTag() Tag {
+ rt := Tag{language: t.locale, locale: t.locale}
+ if t.full == nil {
+ return rt
+ }
+ t, _ = Raw.Compose(rt, t.Variants(), t.Extensions())
+ t, _ = t.SetTypeForKey("rg", "")
+ return t
+}
+
+func compactIndex(t language.Tag) (index compactID, exact bool) {
+ // TODO: perhaps give more frequent tags a lower index.
+ // TODO: we could make the indexes stable. This will excluded some
+ // possibilities for optimization, so don't do this quite yet.
+ exact = true
+
+ b, s, r := t.Raw()
+ switch {
+ case t.HasString():
+ if t.IsPrivateUse() {
+ // We have no entries for user-defined tags.
+ return 0, false
+ }
+ hasExtra := false
+ if t.HasVariants() {
+ if t.HasExtensions() {
+ build := language.Builder{}
+ build.SetTag(language.Tag{LangID: b, ScriptID: s, RegionID: r})
+ build.AddVariant(t.Variants())
+ exact = false
+ t = build.Make()
+ }
+ hasExtra = true
+ } else if _, ok := t.Extension('u'); ok {
+ // TODO: va may mean something else. Consider not considering it.
+ // Strip all but the 'va' entry.
+ old := t
+ variant := t.TypeForKey("va")
+ t = language.Tag{LangID: b, ScriptID: s, RegionID: r}
+ if variant != "" {
+ t, _ = t.SetTypeForKey("va", variant)
+ hasExtra = true
+ }
+ exact = old == t
+ } else {
+ exact = false
+ }
+ if hasExtra {
+ // We have some variants.
+ for i, s := range specialTags {
+ if s == t {
+ return compactID(i + len(coreTags)), exact
+ }
+ }
+ exact = false
+ }
+ }
+ if x, ok := getCoreIndex(t); ok {
+ return x, exact
+ }
+ exact = false
+ if r != 0 && s == 0 {
+ // Deal with cases where an extra script is inserted for the region.
+ t, _ := t.Maximize()
+ if x, ok := getCoreIndex(t); ok {
+ return x, exact
+ }
+ }
+ for t = t.Parent(); t != root; t = t.Parent() {
+ // No variants specified: just compare core components.
+ // The key has the form lllssrrr, where l, s, and r are nibbles for
+ // respectively the langID, scriptID, and regionID.
+ if x, ok := getCoreIndex(t); ok {
+ return x, exact
+ }
+ }
+ return 0, exact
+}
+
+var root = language.Tag{}
+
+// Base is an ISO 639 language code, used for encoding the base language
+// of a language tag.
+type Base struct {
+ langID language.Language
+}
+
+// ParseBase parses a 2- or 3-letter ISO 639 code.
+// It returns a ValueError if s is a well-formed but unknown language identifier
+// or another error if another error occurred.
+func ParseBase(s string) (Base, error) {
+ l, err := language.ParseBase(s)
+ return Base{l}, err
+}
+
+// String returns the BCP 47 representation of the base language.
+func (b Base) String() string {
+ return b.langID.String()
+}
+
+// ISO3 returns the ISO 639-3 language code.
+func (b Base) ISO3() string {
+ return b.langID.ISO3()
+}
+
+// IsPrivateUse reports whether this language code is reserved for private use.
+func (b Base) IsPrivateUse() bool {
+ return b.langID.IsPrivateUse()
+}
+
+// Script is a 4-letter ISO 15924 code for representing scripts.
+// It is idiomatically represented in title case.
+type Script struct {
+ scriptID language.Script
+}
+
+// ParseScript parses a 4-letter ISO 15924 code.
+// It returns a ValueError if s is a well-formed but unknown script identifier
+// or another error if another error occurred.
+func ParseScript(s string) (Script, error) {
+ sc, err := language.ParseScript(s)
+ return Script{sc}, err
+}
+
+// String returns the script code in title case.
+// It returns "Zzzz" for an unspecified script.
+func (s Script) String() string {
+ return s.scriptID.String()
+}
+
+// IsPrivateUse reports whether this script code is reserved for private use.
+func (s Script) IsPrivateUse() bool {
+ return s.scriptID.IsPrivateUse()
+}
+
+// Region is an ISO 3166-1 or UN M.49 code for representing countries and regions.
+type Region struct {
+ regionID language.Region
+}
+
+// EncodeM49 returns the Region for the given UN M.49 code.
+// It returns an error if r is not a valid code.
+func EncodeM49(r int) (Region, error) {
+ rid, err := language.EncodeM49(r)
+ return Region{rid}, err
+}
+
+// ParseRegion parses a 2- or 3-letter ISO 3166-1 or a UN M.49 code.
+// It returns a ValueError if s is a well-formed but unknown region identifier
+// or another error if another error occurred.
+func ParseRegion(s string) (Region, error) {
+ r, err := language.ParseRegion(s)
+ return Region{r}, err
+}
+
+// String returns the BCP 47 representation for the region.
+// It returns "ZZ" for an unspecified region.
+func (r Region) String() string {
+ return r.regionID.String()
+}
+
+// ISO3 returns the 3-letter ISO code of r.
+// Note that not all regions have a 3-letter ISO code.
+// In such cases this method returns "ZZZ".
+func (r Region) ISO3() string {
+ return r.regionID.String()
+}
+
+// M49 returns the UN M.49 encoding of r, or 0 if this encoding
+// is not defined for r.
+func (r Region) M49() int {
+ return r.regionID.M49()
+}
+
+// IsPrivateUse reports whether r has the ISO 3166 User-assigned status. This
+// may include private-use tags that are assigned by CLDR and used in this
+// implementation. So IsPrivateUse and IsCountry can be simultaneously true.
+func (r Region) IsPrivateUse() bool {
+ return r.regionID.IsPrivateUse()
+}
+
+// IsCountry returns whether this region is a country or autonomous area. This
+// includes non-standard definitions from CLDR.
+func (r Region) IsCountry() bool {
+ return r.regionID.IsCountry()
+}
+
+// IsGroup returns whether this region defines a collection of regions. This
+// includes non-standard definitions from CLDR.
+func (r Region) IsGroup() bool {
+ return r.regionID.IsGroup()
+}
+
+// Contains returns whether Region c is contained by Region r. It returns true
+// if c == r.
+func (r Region) Contains(c Region) bool {
+ return r.regionID.Contains(c.regionID)
+}
+
+// TLD returns the country code top-level domain (ccTLD). UK is returned for GB.
+// In all other cases it returns either the region itself or an error.
+//
+// This method may return an error for a region for which there exists a
+// canonical form with a ccTLD. To get that ccTLD canonicalize r first. The
+// region will already be canonicalized it was obtained from a Tag that was
+// obtained using any of the default methods.
+func (r Region) TLD() (Region, error) {
+ tld, err := r.regionID.TLD()
+ return Region{tld}, err
+}
+
+// Canonicalize returns the region or a possible replacement if the region is
+// deprecated. It will not return a replacement for deprecated regions that
+// are split into multiple regions.
+func (r Region) Canonicalize() Region {
+ return Region{r.regionID.Canonicalize()}
+}
+
+// Variant represents a registered variant of a language as defined by BCP 47.
+type Variant struct {
+ variant string
+}
+
+// ParseVariant parses and returns a Variant. An error is returned if s is not
+// a valid variant.
+func ParseVariant(s string) (Variant, error) {
+ v, err := language.ParseVariant(s)
+ return Variant{v.String()}, err
+}
+
+// String returns the string representation of the variant.
+func (v Variant) String() string {
+ return v.variant
+}
diff --git a/internal/language/compact/language_test.go b/internal/language/compact/language_test.go
new file mode 100644
index 0000000..20bc48c
--- /dev/null
+++ b/internal/language/compact/language_test.go
@@ -0,0 +1,844 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+import (
+ "reflect"
+ "testing"
+)
+
+func TestTagSize(t *testing.T) {
+ id := Tag{}
+ typ := reflect.TypeOf(id)
+ if typ.Size() > 24 {
+ t.Errorf("size of Tag was %d; want 24", typ.Size())
+ }
+}
+
+func TestIsRoot(t *testing.T) {
+ loc := Tag{}
+ if !loc.IsRoot() {
+ t.Errorf("unspecified should be root.")
+ }
+ for i, tt := range parseTests() {
+ loc, _ := Parse(tt.in)
+ undef := tt.lang == "und" && tt.script == "" && tt.region == "" && tt.ext == ""
+ if loc.IsRoot() != undef {
+ t.Errorf("%d: was %v; want %v", i, loc.IsRoot(), undef)
+ }
+ }
+}
+
+func TestEquality(t *testing.T) {
+ for i, tt := range parseTests() {
+ s := tt.in
+ tag := Make(s)
+ t1 := Make(tag.String())
+ if tag != t1 {
+ t.Errorf("%d:%s: equality test 1 failed\n got: %#v\nwant: %#v)", i, s, t1, tag)
+ }
+ t2, _ := Compose(tag)
+ if tag != t2 {
+ t.Errorf("%d:%s: equality test 2 failed\n got: %#v\nwant: %#v", i, s, t2, tag)
+ }
+ }
+}
+
+func TestString(t *testing.T) {
+ tests := []string{
+ "no-u-rg-dkzzzz",
+ }
+ for i, s := range tests {
+ tag := Make(s)
+ if tag.String() != s {
+ t.Errorf("%d:%s: got %s: want %s (%#v)", i, s, tag.String(), s, tag)
+ }
+ }
+}
+
+type compactTest struct {
+ tag string
+ index compactID
+ ok bool
+}
+
+var compactTests = []compactTest{
+ // TODO: these values will change with each CLDR update. This issue
+ // will be solved if we decide to fix the indexes.
+ {"und", undIndex, true},
+ {"ca-ES-valencia", caESvalenciaIndex, true},
+ {"ca-ES-valencia-u-va-posix", caESvalenciaIndex, false},
+ {"ca-ES-valencia-u-co-phonebk", caESvalenciaIndex, false},
+ {"ca-ES-valencia-u-co-phonebk-va-posix", caESvalenciaIndex, false},
+ {"x-klingon", 0, false},
+ {"en-US", enUSIndex, true},
+ {"en-US-u-va-posix", enUSuvaposixIndex, true},
+ {"en", enIndex, true},
+ {"en-u-co-phonebk", enIndex, false},
+ {"en-001", en001Index, true},
+ {"zh-Hant-HK", zhHantHKIndex, true},
+ {"zh-HK", zhHantHKIndex, false}, // maximized to zh-Hant-HK
+ {"nl-Beng", 0, false}, // parent skips script
+ {"nl-NO", nlIndex, false}, // region is ignored
+ {"nl-Latn-NO", nlIndex, false},
+ {"nl-Latn-NO-u-co-phonebk", nlIndex, false},
+ {"nl-Latn-NO-valencia", nlIndex, false},
+ {"nl-Latn-NO-oxendict", nlIndex, false},
+ {"sh", shIndex, true}, // From plural rules.
+}
+
+func TestCompactIndex(t *testing.T) {
+ tests := append(compactTests, []compactTest{
+ {"en-GB", enGBIndex, true},
+ {"en-GB-u-rg-uszzzz", enGBIndex, true},
+ {"en-GB-u-rg-USZZZZ", enGBIndex, true},
+ {"en-GB-u-rg-uszzzz-va-posix", enGBIndex, false},
+ {"en-GB-u-co-phonebk-rg-uszzzz", enGBIndex, false},
+ // Invalid region specifications are ignored.
+ {"en-GB-u-rg-usz-va-posix", enGBIndex, false},
+ {"en-GB-u-co-phonebk-rg-usz", enGBIndex, false},
+ }...)
+ for _, tt := range tests {
+ x, ok := CompactIndex(Raw.MustParse(tt.tag))
+ if compactID(x) != tt.index || ok != tt.ok {
+ t.Errorf("%s: got %d, %v; want %d %v", tt.tag, x, ok, tt.index, tt.ok)
+ }
+ }
+}
+
+func TestRegionalCompactIndex(t *testing.T) {
+ tests := append(compactTests, []compactTest{
+ {"en-GB", enGBIndex, true},
+ {"en-GB-u-rg-uszzzz", enUSIndex, true},
+ {"en-GB-u-rg-USZZZZ", enUSIndex, true},
+ // TODO: use different exact values for language and regional tag?
+ {"en-GB-u-rg-uszzzz-va-posix", enUSuvaposixIndex, false},
+ {"en-GB-u-co-phonebk-rg-uszzzz-va-posix", enUSuvaposixIndex, false},
+ {"en-GB-u-co-phonebk-rg-uszzzz", enUSIndex, false},
+ // Invalid region specifications are ignored.
+ {"en-GB-u-rg-usz-va-posix", enGBIndex, false},
+ {"en-GB-u-co-phonebk-rg-usz", enGBIndex, false},
+ }...)
+ for _, tt := range tests {
+ x, ok := regionalCompactIndex(Raw.MustParse(tt.tag))
+ if compactID(x) != tt.index || ok != tt.ok {
+ t.Errorf("%s: got %d, %v; want %d %v", tt.tag, x, ok, tt.index, tt.ok)
+ }
+ }
+}
+
+func TestMarshal(t *testing.T) {
+ testCases := []string{
+ // TODO: these values will change with each CLDR update. This issue
+ // will be solved if we decide to fix the indexes.
+ "und",
+ "ca-ES-valencia",
+ "ca-ES-valencia-u-va-posix",
+ "ca-ES-valencia-u-co-phonebk",
+ "ca-ES-valencia-u-co-phonebk-va-posix",
+ "x-klingon",
+ "en-US",
+ "en-US-u-va-posix",
+ "en",
+ "en-u-co-phonebk",
+ "en-001",
+ "sh",
+
+ "en-GB-u-rg-uszzzz",
+ "en-GB-u-rg-uszzzz-va-posix",
+ "en-GB-u-co-phonebk-rg-uszzzz",
+ // Invalid tags should also roundtrip.
+ "en-GB-u-co-phonebk-rg-uszz",
+ }
+ for _, tc := range testCases {
+ var tag Tag
+ err := tag.UnmarshalText([]byte(tc))
+ if err != nil {
+ t.Errorf("UnmarshalText(%q): unexpected error: %v", tc, err)
+ }
+ b, err := tag.MarshalText()
+ if err != nil {
+ t.Errorf("MarshalText(%q): unexpected error: %v", tc, err)
+ }
+ if got := string(b); got != tc {
+ t.Errorf("%s: got %q; want %q", tc, got, tc)
+ }
+ }
+}
+
+func TestBase(t *testing.T) {
+ tests := []struct {
+ loc, lang string
+ conf Confidence
+ }{
+ {"und", "en", Low},
+ {"x-abc", "und", No},
+ {"en", "en", Exact},
+ {"und-Cyrl", "ru", High},
+ // If a region is not included, the official language should be English.
+ {"und-US", "en", High},
+ // TODO: not-explicitly listed scripts should probably be und, No
+ // Modify addTags to return info on how the match was derived.
+ // {"und-Aghb", "und", No},
+ }
+ for i, tt := range tests {
+ loc, _ := Parse(tt.loc)
+ lang, conf := loc.Base()
+ if lang.String() != tt.lang {
+ t.Errorf("%d: language was %s; want %s", i, lang, tt.lang)
+ }
+ if conf != tt.conf {
+ t.Errorf("%d: confidence was %d; want %d", i, conf, tt.conf)
+ }
+ }
+}
+
+func TestParseBase(t *testing.T) {
+ tests := []struct {
+ in string
+ out string
+ ok bool
+ }{
+ {"en", "en", true},
+ {"EN", "en", true},
+ {"nld", "nl", true},
+ {"dut", "dut", true}, // bibliographic
+ {"aaj", "und", false}, // unknown
+ {"qaa", "qaa", true},
+ {"a", "und", false},
+ {"", "und", false},
+ {"aaaa", "und", false},
+ }
+ for i, tt := range tests {
+ x, err := ParseBase(tt.in)
+ if x.String() != tt.out || err == nil != tt.ok {
+ t.Errorf("%d:%s: was %s, %v; want %s, %v", i, tt.in, x, err == nil, tt.out, tt.ok)
+ }
+ if y, _, _ := Raw.Make(tt.out).Raw(); x != y {
+ t.Errorf("%d:%s: tag was %s; want %s", i, tt.in, x, y)
+ }
+ }
+}
+
+func TestScript(t *testing.T) {
+ tests := []struct {
+ loc, scr string
+ conf Confidence
+ }{
+ {"und", "Latn", Low},
+ {"en-Latn", "Latn", Exact},
+ {"en", "Latn", High},
+ {"sr", "Cyrl", Low},
+ {"kk", "Cyrl", High},
+ {"kk-CN", "Arab", Low},
+ {"cmn", "Hans", Low},
+ {"ru", "Cyrl", High},
+ {"ru-RU", "Cyrl", High},
+ {"yue", "Hant", Low},
+ {"x-abc", "Zzzz", Low},
+ {"und-zyyy", "Zyyy", Exact},
+ }
+ for i, tt := range tests {
+ loc, _ := Parse(tt.loc)
+ sc, conf := loc.Script()
+ if sc.String() != tt.scr {
+ t.Errorf("%d:%s: script was %s; want %s", i, tt.loc, sc, tt.scr)
+ }
+ if conf != tt.conf {
+ t.Errorf("%d:%s: confidence was %d; want %d", i, tt.loc, conf, tt.conf)
+ }
+ }
+}
+
+func TestParseScript(t *testing.T) {
+ tests := []struct {
+ in string
+ out string
+ ok bool
+ }{
+ {"Latn", "Latn", true},
+ {"zzzz", "Zzzz", true},
+ {"zyyy", "Zyyy", true},
+ {"Latm", "Zzzz", false},
+ {"Zzz", "Zzzz", false},
+ {"", "Zzzz", false},
+ {"Zzzxx", "Zzzz", false},
+ }
+ for i, tt := range tests {
+ x, err := ParseScript(tt.in)
+ if x.String() != tt.out || err == nil != tt.ok {
+ t.Errorf("%d:%s: was %s, %v; want %s, %v", i, tt.in, x, err == nil, tt.out, tt.ok)
+ }
+ if err == nil {
+ if _, y, _ := Raw.Make("und-" + tt.out).Raw(); x != y {
+ t.Errorf("%d:%s: tag was %s; want %s", i, tt.in, x, y)
+ }
+ }
+ }
+}
+
+func TestRegion(t *testing.T) {
+ tests := []struct {
+ loc, reg string
+ conf Confidence
+ }{
+ {"und", "US", Low},
+ {"en", "US", Low},
+ {"zh-Hant", "TW", Low},
+ {"en-US", "US", Exact},
+ {"cmn", "CN", Low},
+ {"ru", "RU", Low},
+ {"yue", "HK", Low},
+ {"x-abc", "ZZ", Low},
+ }
+ for i, tt := range tests {
+ loc, _ := Raw.Parse(tt.loc)
+ reg, conf := loc.Region()
+ if reg.String() != tt.reg {
+ t.Errorf("%d:%s: region was %s; want %s", i, tt.loc, reg, tt.reg)
+ }
+ if conf != tt.conf {
+ t.Errorf("%d:%s: confidence was %d; want %d", i, tt.loc, conf, tt.conf)
+ }
+ }
+}
+
+func TestEncodeM49(t *testing.T) {
+ tests := []struct {
+ m49 int
+ code string
+ ok bool
+ }{
+ {1, "001", true},
+ {840, "US", true},
+ {899, "ZZ", false},
+ }
+ for i, tt := range tests {
+ if r, err := EncodeM49(tt.m49); r.String() != tt.code || err == nil != tt.ok {
+ t.Errorf("%d:%d: was %s, %v; want %s, %v", i, tt.m49, r, err == nil, tt.code, tt.ok)
+ }
+ }
+ for i := 1; i <= 1000; i++ {
+ if r, err := EncodeM49(i); err == nil && r.M49() == 0 {
+ t.Errorf("%d has no error, but maps to undefined region", i)
+ }
+ }
+}
+
+func TestParseRegion(t *testing.T) {
+ tests := []struct {
+ in string
+ out string
+ ok bool
+ }{
+ {"001", "001", true},
+ {"840", "US", true},
+ {"899", "ZZ", false},
+ {"USA", "US", true},
+ {"US", "US", true},
+ {"BC", "ZZ", false},
+ {"C", "ZZ", false},
+ {"CCCC", "ZZ", false},
+ {"01", "ZZ", false},
+ }
+ for i, tt := range tests {
+ r, err := ParseRegion(tt.in)
+ if r.String() != tt.out || err == nil != tt.ok {
+ t.Errorf("%d:%s: was %s, %v; want %s, %v", i, tt.in, r, err == nil, tt.out, tt.ok)
+ }
+ if err == nil {
+ if _, _, y := Raw.Make("und-" + tt.out).Raw(); r != y {
+ t.Errorf("%d:%s: tag was %s; want %s", i, tt.in, r, y)
+ }
+ }
+ }
+}
+
+func TestIsCountry(t *testing.T) {
+ tests := []struct {
+ reg string
+ country bool
+ }{
+ {"US", true},
+ {"001", false},
+ {"958", false},
+ {"419", false},
+ {"203", true},
+ {"020", true},
+ {"900", false},
+ {"999", false},
+ {"QO", false},
+ {"EU", false},
+ {"AA", false},
+ {"XK", true},
+ }
+ for i, tt := range tests {
+ r, _ := ParseRegion(tt.reg)
+ if r.IsCountry() != tt.country {
+ t.Errorf("%d: IsCountry(%s) was %v; want %v", i, tt.reg, r.IsCountry(), tt.country)
+ }
+ }
+}
+
+func TestIsGroup(t *testing.T) {
+ tests := []struct {
+ reg string
+ group bool
+ }{
+ {"US", false},
+ {"001", true},
+ {"958", false},
+ {"419", true},
+ {"203", false},
+ {"020", false},
+ {"900", false},
+ {"999", false},
+ {"QO", true},
+ {"EU", true},
+ {"AA", false},
+ {"XK", false},
+ }
+ for i, tt := range tests {
+ r, _ := ParseRegion(tt.reg)
+ if r.IsGroup() != tt.group {
+ t.Errorf("%d: IsGroup(%s) was %v; want %v", i, tt.reg, r.IsGroup(), tt.group)
+ }
+ }
+}
+
+func TestContains(t *testing.T) {
+ tests := []struct {
+ enclosing, contained string
+ contains bool
+ }{
+ // A region contains itself.
+ {"US", "US", true},
+ {"001", "001", true},
+
+ // Direct containment.
+ {"001", "002", true},
+ {"039", "XK", true},
+ {"150", "XK", true},
+ {"EU", "AT", true},
+ {"QO", "AQ", true},
+
+ // Indirect containemnt.
+ {"001", "US", true},
+ {"001", "419", true},
+ {"001", "013", true},
+
+ // No containment.
+ {"US", "001", false},
+ {"155", "EU", false},
+ }
+ for i, tt := range tests {
+ r := MustParseRegion(tt.enclosing)
+ con := MustParseRegion(tt.contained)
+ if got := r.Contains(con); got != tt.contains {
+ t.Errorf("%d: %s.Contains(%s) was %v; want %v", i, tt.enclosing, tt.contained, got, tt.contains)
+ }
+ }
+}
+
+func TestRegionCanonicalize(t *testing.T) {
+ for i, tt := range []struct{ in, out string }{
+ {"UK", "GB"},
+ {"TP", "TL"},
+ {"QU", "EU"},
+ {"SU", "SU"},
+ {"VD", "VN"},
+ {"DD", "DE"},
+ } {
+ r := MustParseRegion(tt.in)
+ want := MustParseRegion(tt.out)
+ if got := r.Canonicalize(); got != want {
+ t.Errorf("%d: got %v; want %v", i, got, want)
+ }
+ }
+}
+
+func TestRegionTLD(t *testing.T) {
+ for _, tt := range []struct {
+ in, out string
+ ok bool
+ }{
+ {"EH", "EH", true},
+ {"FR", "FR", true},
+ {"TL", "TL", true},
+
+ // In ccTLD before in ISO.
+ {"GG", "GG", true},
+
+ // Non-standard assignment of ccTLD to ISO code.
+ {"GB", "UK", true},
+
+ // Exceptionally reserved in ISO and valid ccTLD.
+ {"UK", "UK", true},
+ {"AC", "AC", true},
+ {"EU", "EU", true},
+ {"SU", "SU", true},
+
+ // Exceptionally reserved in ISO and invalid ccTLD.
+ {"CP", "ZZ", false},
+ {"DG", "ZZ", false},
+ {"EA", "ZZ", false},
+ {"FX", "ZZ", false},
+ {"IC", "ZZ", false},
+ {"TA", "ZZ", false},
+
+ // Transitionally reserved in ISO (e.g. deprecated) but valid ccTLD as
+ // it is still being phased out.
+ {"AN", "AN", true},
+ {"TP", "TP", true},
+
+ // Transitionally reserved in ISO (e.g. deprecated) and invalid ccTLD.
+ // Defined in package language as it has a mapping in CLDR.
+ {"BU", "ZZ", false},
+ {"CS", "ZZ", false},
+ {"NT", "ZZ", false},
+ {"YU", "ZZ", false},
+ {"ZR", "ZZ", false},
+ // Not defined in package: SF.
+
+ // Indeterminately reserved in ISO.
+ // Defined in package language as it has a legacy mapping in CLDR.
+ {"DY", "ZZ", false},
+ {"RH", "ZZ", false},
+ {"VD", "ZZ", false},
+ // Not defined in package: EW, FL, JA, LF, PI, RA, RB, RC, RI, RL, RM,
+ // RN, RP, WG, WL, WV, and YV.
+
+ // Not assigned in ISO, but legacy definitions in CLDR.
+ {"DD", "ZZ", false},
+ {"YD", "ZZ", false},
+
+ // Normal mappings but somewhat special status in ccTLD.
+ {"BL", "BL", true},
+ {"MF", "MF", true},
+ {"BV", "BV", true},
+ {"SJ", "SJ", true},
+
+ // Have values when normalized, but not as is.
+ {"QU", "ZZ", false},
+
+ // ISO Private Use.
+ {"AA", "ZZ", false},
+ {"QM", "ZZ", false},
+ {"QO", "ZZ", false},
+ {"XA", "ZZ", false},
+ {"XK", "ZZ", false}, // Sometimes used for Kosovo, but invalid ccTLD.
+ } {
+ if tt.in == "" {
+ continue
+ }
+
+ r := MustParseRegion(tt.in)
+ var want Region
+ if tt.out != "ZZ" {
+ want = MustParseRegion(tt.out)
+ }
+ tld, err := r.TLD()
+ if got := err == nil; got != tt.ok {
+ t.Errorf("error(%v): got %v; want %v", r, got, tt.ok)
+ }
+ if tld != want {
+ t.Errorf("TLD(%v): got %v; want %v", r, tld, want)
+ }
+ }
+}
+
+func TestCanonicalize(t *testing.T) {
+ // TODO: do a full test using CLDR data in a separate regression test.
+ tests := []struct {
+ in, out string
+ option CanonType
+ }{
+ {"en-Latn", "en", SuppressScript},
+ {"sr-Cyrl", "sr-Cyrl", SuppressScript},
+ {"sh", "sr-Latn", Legacy},
+ {"sh-HR", "sr-Latn-HR", Legacy},
+ {"sh-Cyrl-HR", "sr-Cyrl-HR", Legacy},
+ {"tl", "fil", Legacy},
+ {"no", "no", Legacy},
+ {"no", "nb", Legacy | CLDR},
+ {"cmn", "cmn", Legacy},
+ {"cmn", "zh", Macro},
+ {"cmn-u-co-stroke", "zh-u-co-stroke", Macro},
+ {"yue", "yue", Macro},
+ {"nb", "no", Macro},
+ {"nb", "nb", Macro | CLDR},
+ {"no", "no", Macro},
+ {"no", "no", Macro | CLDR},
+ {"iw", "he", DeprecatedBase},
+ {"iw", "he", Deprecated | CLDR},
+ {"mo", "ro-MD", Deprecated}, // Adopted by CLDR as of version 25.
+ {"alb", "sq", Legacy}, // bibliographic
+ {"dut", "nl", Legacy}, // bibliographic
+ // As of CLDR 25, mo is no longer considered a legacy mapping.
+ {"mo", "mo", Legacy | CLDR},
+ {"und-AN", "und-AN", Deprecated},
+ {"und-YD", "und-YE", DeprecatedRegion},
+ {"und-YD", "und-YD", DeprecatedBase},
+ {"und-Qaai", "und-Zinh", DeprecatedScript},
+ {"und-Qaai", "und-Qaai", DeprecatedBase},
+ {"drh", "mn", All}, // drh -> khk -> mn
+
+ {"en-GB-u-rg-uszzzz", "en-GB-u-rg-uszzzz", Raw},
+ {"en-GB-u-rg-USZZZZ", "en-GB-u-rg-uszzzz", Raw},
+ // TODO: use different exact values for language and regional tag?
+ {"en-GB-u-rg-uszzzz-va-posix", "en-GB-u-rg-uszzzz-va-posix", Raw},
+ {"en-GB-u-rg-uszzzz-co-phonebk", "en-GB-u-co-phonebk-rg-uszzzz", Raw},
+ // Invalid region specifications are left as is.
+ {"en-GB-u-rg-usz", "en-GB-u-rg-usz", Raw},
+ {"en-GB-u-rg-usz-va-posix", "en-GB-u-rg-usz-va-posix", Raw},
+ {"en-GB-u-rg-usz-co-phonebk", "en-GB-u-co-phonebk-rg-usz", Raw},
+ }
+ for i, tt := range tests {
+ in, _ := Raw.Parse(tt.in)
+ in, _ = tt.option.Canonicalize(in)
+ if in.String() != tt.out {
+ t.Errorf("%d:%s: was %s; want %s", i, tt.in, in.String(), tt.out)
+ }
+ }
+ // Test idempotence.
+ for _, base := range Supported.BaseLanguages() {
+ tag, _ := Raw.Compose(base)
+ got, _ := All.Canonicalize(tag)
+ want, _ := All.Canonicalize(got)
+ if got != want {
+ t.Errorf("idem(%s): got %s; want %s", tag, got, want)
+ }
+ }
+}
+
+func TestTypeForKey(t *testing.T) {
+ tests := []struct{ key, in, out string }{
+ {"co", "en", ""},
+ {"co", "en-u-abc", ""},
+ {"co", "en-u-co-phonebk", "phonebk"},
+ {"co", "en-u-co-phonebk-cu-aud", "phonebk"},
+ {"co", "x-foo-u-co-phonebk", ""},
+ {"va", "en-US-u-va-posix", "posix"},
+ {"rg", "en-u-rg-gbzzzz", "gbzzzz"},
+ {"nu", "en-u-co-phonebk-nu-arabic", "arabic"},
+ {"kc", "cmn-u-co-stroke", ""},
+ }
+ for _, tt := range tests {
+ if v := Make(tt.in).TypeForKey(tt.key); v != tt.out {
+ t.Errorf("%q[%q]: was %q; want %q", tt.in, tt.key, v, tt.out)
+ }
+ }
+}
+
+func TestParent(t *testing.T) {
+ tests := []struct{ in, out string }{
+ // Strip variants and extensions first
+ {"de-u-co-phonebk", "de"},
+ {"de-1994", "de"},
+ {"de-Latn-1994", "de"}, // remove superfluous script.
+
+ // Ensure the canonical Tag for an entry is in the chain for base-script
+ // pairs.
+ {"zh-Hans", "zh"},
+
+ // Skip the script if it is the maximized version. CLDR files for the
+ // skipped tag are always empty.
+ {"zh-Hans-TW", "zh"},
+ {"zh-Hans-CN", "zh"},
+
+ // Insert the script if the maximized script is not the same as the
+ // maximized script of the base language.
+ {"zh-TW", "zh-Hant"},
+ {"zh-HK", "zh-Hant"},
+ {"zh-Hant-TW", "zh-Hant"},
+ {"zh-Hant-HK", "zh-Hant"},
+
+ // Non-default script skips to und.
+ // CLDR
+ {"az-Cyrl", "und"},
+ {"bs-Cyrl", "und"},
+ {"en-Dsrt", "und"},
+ {"ha-Arab", "und"},
+ {"mn-Mong", "und"},
+ {"pa-Arab", "und"},
+ {"shi-Latn", "und"},
+ {"sr-Latn", "und"},
+ {"uz-Arab", "und"},
+ {"uz-Cyrl", "und"},
+ {"vai-Latn", "und"},
+ {"zh-Hant", "und"},
+ // extra
+ {"nl-Cyrl", "und"},
+
+ // World english inherits from en-001.
+ {"en-150", "en-001"},
+ {"en-AU", "en-001"},
+ {"en-BE", "en-001"},
+ {"en-GG", "en-001"},
+ {"en-GI", "en-001"},
+ {"en-HK", "en-001"},
+ {"en-IE", "en-001"},
+ {"en-IM", "en-001"},
+ {"en-IN", "en-001"},
+ {"en-JE", "en-001"},
+ {"en-MT", "en-001"},
+ {"en-NZ", "en-001"},
+ {"en-PK", "en-001"},
+ {"en-SG", "en-001"},
+
+ // Spanish in Latin-American countries have es-419 as parent.
+ {"es-AR", "es-419"},
+ {"es-BO", "es-419"},
+ {"es-CL", "es-419"},
+ {"es-CO", "es-419"},
+ {"es-CR", "es-419"},
+ {"es-CU", "es-419"},
+ {"es-DO", "es-419"},
+ {"es-EC", "es-419"},
+ {"es-GT", "es-419"},
+ {"es-HN", "es-419"},
+ {"es-MX", "es-419"},
+ {"es-NI", "es-419"},
+ {"es-PA", "es-419"},
+ {"es-PE", "es-419"},
+ {"es-PR", "es-419"},
+ {"es-PY", "es-419"},
+ {"es-SV", "es-419"},
+ {"es-US", "es-419"},
+ {"es-UY", "es-419"},
+ {"es-VE", "es-419"},
+ // exceptions (according to CLDR)
+ {"es-CW", "es"},
+
+ // Inherit from pt-PT, instead of pt for these countries.
+ {"pt-AO", "pt-PT"},
+ {"pt-CV", "pt-PT"},
+ {"pt-GW", "pt-PT"},
+ {"pt-MO", "pt-PT"},
+ {"pt-MZ", "pt-PT"},
+ {"pt-ST", "pt-PT"},
+ {"pt-TL", "pt-PT"},
+
+ {"en-GB-u-co-phonebk-rg-uszzzz", "en-GB"},
+ {"en-GB-u-rg-uszzzz", "en-GB"},
+ {"en-US-u-va-posix", "en-US"},
+
+ // Difference between language and regional tag.
+ {"ca-ES-valencia", "ca-ES"},
+ {"ca-ES-valencia-u-rg-ptzzzz", "ca-ES"}, // t.full != nil
+ {"en-US-u-va-variant", "en-US"},
+ {"en-u-va-variant", "en"}, // t.full != nil
+ {"en-u-rg-gbzzzz", "en"},
+ {"en-US-u-rg-gbzzzz", "en-US"},
+ {"nl-US-u-rg-gbzzzz", "nl-US"}, // t.full != nil
+ }
+ for _, tt := range tests {
+ tag := Raw.MustParse(tt.in)
+ if p := Raw.MustParse(tt.out); p != tag.Parent() {
+ t.Errorf("%s: was %v; want %v", tt.in, tag.Parent(), p)
+ }
+ }
+}
+
+var (
+ // Tags without error that don't need to be changed.
+ benchBasic = []string{
+ "en",
+ "en-Latn",
+ "en-GB",
+ "za",
+ "zh-Hant",
+ "zh",
+ "zh-HK",
+ "ar-MK",
+ "en-CA",
+ "fr-CA",
+ "fr-CH",
+ "fr",
+ "lv",
+ "he-IT",
+ "tlh",
+ "ja",
+ "ja-Jpan",
+ "ja-Jpan-JP",
+ "de-1996",
+ "de-CH",
+ "sr",
+ "sr-Latn",
+ }
+ // Tags with extensions, not changes required.
+ benchExt = []string{
+ "x-a-b-c-d",
+ "x-aa-bbbb-cccccccc-d",
+ "en-x_cc-b-bbb-a-aaa",
+ "en-c_cc-b-bbb-a-aaa-x-x",
+ "en-u-co-phonebk",
+ "en-Cyrl-u-co-phonebk",
+ "en-US-u-co-phonebk-cu-xau",
+ "en-nedix-u-co-phonebk",
+ "en-t-t0-abcd",
+ "en-t-nl-latn",
+ "en-t-t0-abcd-x-a",
+ }
+ // Change, but not memory allocation required.
+ benchSimpleChange = []string{
+ "EN",
+ "i-klingon",
+ "en-latn",
+ "zh-cmn-Hans-CN",
+ "iw-NL",
+ }
+ // Change and memory allocation required.
+ benchChangeAlloc = []string{
+ "en-c_cc-b-bbb-a-aaa",
+ "en-u-cu-xua-co-phonebk",
+ "en-u-cu-xua-co-phonebk-a-cd",
+ "en-u-def-abc-cu-xua-co-phonebk",
+ "en-t-en-Cyrl-NL-1994",
+ "en-t-en-Cyrl-NL-1994-t0-abc-def",
+ }
+ // Tags that result in errors.
+ benchErr = []string{
+ // IllFormed
+ "x_A.-B-C_D",
+ "en-u-cu-co-phonebk",
+ "en-u-cu-xau-co",
+ "en-t-nl-abcd",
+ // Invalid
+ "xx",
+ "nl-Uuuu",
+ "nl-QB",
+ }
+ benchChange = append(benchSimpleChange, benchChangeAlloc...)
+ benchAll = append(append(append(benchBasic, benchExt...), benchChange...), benchErr...)
+)
+
+func doParse(b *testing.B, tag []string) {
+ for i := 0; i < b.N; i++ {
+ // Use the modulo instead of looping over all tags so that we get a somewhat
+ // meaningful ns/op.
+ Parse(tag[i%len(tag)])
+ }
+}
+
+func BenchmarkParse(b *testing.B) {
+ doParse(b, benchAll)
+}
+
+func BenchmarkParseBasic(b *testing.B) {
+ doParse(b, benchBasic)
+}
+
+func BenchmarkParseError(b *testing.B) {
+ doParse(b, benchErr)
+}
+
+func BenchmarkParseSimpleChange(b *testing.B) {
+ doParse(b, benchSimpleChange)
+}
+
+func BenchmarkParseChangeAlloc(b *testing.B) {
+ doParse(b, benchChangeAlloc)
+}
diff --git a/internal/language/compact/parse.go b/internal/language/compact/parse.go
new file mode 100644
index 0000000..d50c8aa
--- /dev/null
+++ b/internal/language/compact/parse.go
@@ -0,0 +1,228 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+import (
+ "errors"
+ "strconv"
+ "strings"
+
+ "golang.org/x/text/internal/language"
+)
+
+// ValueError is returned by any of the parsing functions when the
+// input is well-formed but the respective subtag is not recognized
+// as a valid value.
+type ValueError interface {
+ error
+
+ // Subtag returns the subtag for which the error occurred.
+ Subtag() string
+}
+
+// Parse parses the given BCP 47 string and returns a valid Tag. If parsing
+// failed it returns an error and any part of the tag that could be parsed.
+// If parsing succeeded but an unknown value was found, it returns
+// ValueError. The Tag returned in this case is just stripped of the unknown
+// value. All other values are preserved. It accepts tags in the BCP 47 format
+// and extensions to this standard defined in
+// http://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
+// The resulting tag is canonicalized using the default canonicalization type.
+func Parse(s string) (t Tag, err error) {
+ return Default.Parse(s)
+}
+
+// Parse parses the given BCP 47 string and returns a valid Tag. If parsing
+// failed it returns an error and any part of the tag that could be parsed.
+// If parsing succeeded but an unknown value was found, it returns
+// ValueError. The Tag returned in this case is just stripped of the unknown
+// value. All other values are preserved. It accepts tags in the BCP 47 format
+// and extensions to this standard defined in
+// http://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
+// The resulting tag is canonicalized using the the canonicalization type c.
+func (c CanonType) Parse(s string) (t Tag, err error) {
+ tt, err := language.Parse(s)
+ if err != nil {
+ return makeTag(tt), err
+ }
+ tt, changed := canonicalize(c, tt)
+ if changed {
+ tt.RemakeString()
+ }
+ return makeTag(tt), err
+}
+
+// Compose creates a Tag from individual parts, which may be of type Tag, Base,
+// Script, Region, Variant, []Variant, Extension, []Extension or error. If a
+// Base, Script or Region or slice of type Variant or Extension is passed more
+// than once, the latter will overwrite the former. Variants and Extensions are
+// accumulated, but if two extensions of the same type are passed, the latter
+// will replace the former. For -u extensions, though, the key-type pairs are
+// added, where later values overwrite older ones. A Tag overwrites all former
+// values and typically only makes sense as the first argument. The resulting
+// tag is returned after canonicalizing using the Default CanonType. If one or
+// more errors are encountered, one of the errors is returned.
+func Compose(part ...interface{}) (t Tag, err error) {
+ return Default.Compose(part...)
+}
+
+// Compose creates a Tag from individual parts, which may be of type Tag, Base,
+// Script, Region, Variant, []Variant, Extension, []Extension or error. If a
+// Base, Script or Region or slice of type Variant or Extension is passed more
+// than once, the latter will overwrite the former. Variants and Extensions are
+// accumulated, but if two extensions of the same type are passed, the latter
+// will replace the former. For -u extensions, though, the key-type pairs are
+// added, where later values overwrite older ones. A Tag overwrites all former
+// values and typically only makes sense as the first argument. The resulting
+// tag is returned after canonicalizing using CanonType c. If one or more errors
+// are encountered, one of the errors is returned.
+func (c CanonType) Compose(part ...interface{}) (t Tag, err error) {
+ var b language.Builder
+ if err = update(&b, part...); err != nil {
+ return und, err
+ }
+ b.Tag, _ = canonicalize(c, b.Tag)
+ return makeTag(b.Make()), err
+}
+
+var errInvalidArgument = errors.New("invalid Extension or Variant")
+
+func update(b *language.Builder, part ...interface{}) (err error) {
+ for _, x := range part {
+ switch v := x.(type) {
+ case Tag:
+ b.SetTag(v.tag())
+ case Base:
+ b.Tag.LangID = v.langID
+ case Script:
+ b.Tag.ScriptID = v.scriptID
+ case Region:
+ b.Tag.RegionID = v.regionID
+ case Variant:
+ if v.variant == "" {
+ err = errInvalidArgument
+ break
+ }
+ b.AddVariant(v.variant)
+ case Extension:
+ if v.s == "" {
+ err = errInvalidArgument
+ break
+ }
+ b.SetExt(v.s)
+ case []Variant:
+ b.ClearVariants()
+ for _, v := range v {
+ b.AddVariant(v.variant)
+ }
+ case []Extension:
+ b.ClearExtensions()
+ for _, e := range v {
+ b.SetExt(e.s)
+ }
+ // TODO: support parsing of raw strings based on morphology or just extensions?
+ case error:
+ if v != nil {
+ err = v
+ }
+ }
+ }
+ return
+}
+
+var errInvalidWeight = errors.New("ParseAcceptLanguage: invalid weight")
+
+// ParseAcceptLanguage parses the contents of an Accept-Language header as
+// defined in http://www.ietf.org/rfc/rfc2616.txt and returns a list of Tags and
+// a list of corresponding quality weights. It is more permissive than RFC 2616
+// and may return non-nil slices even if the input is not valid.
+// The Tags will be sorted by highest weight first and then by first occurrence.
+// Tags with a weight of zero will be dropped. An error will be returned if the
+// input could not be parsed.
+func ParseAcceptLanguage(s string) (tag []Tag, q []float32, err error) {
+ var entry string
+ for s != "" {
+ if entry, s = split(s, ','); entry == "" {
+ continue
+ }
+
+ entry, weight := split(entry, ';')
+
+ // Scan the language.
+ t, err := Parse(entry)
+ if err != nil {
+ id, ok := acceptFallback[entry]
+ if !ok {
+ return nil, nil, err
+ }
+ t = makeTag(language.Tag{LangID: id})
+ }
+
+ // Scan the optional weight.
+ w := 1.0
+ if weight != "" {
+ weight = consume(weight, 'q')
+ weight = consume(weight, '=')
+ // consume returns the empty string when a token could not be
+ // consumed, resulting in an error for ParseFloat.
+ if w, err = strconv.ParseFloat(weight, 32); err != nil {
+ return nil, nil, errInvalidWeight
+ }
+ // Drop tags with a quality weight of 0.
+ if w <= 0 {
+ continue
+ }
+ }
+
+ tag = append(tag, t)
+ q = append(q, float32(w))
+ }
+ sortStable(&tagSort{tag, q})
+ return tag, q, nil
+}
+
+// consume removes a leading token c from s and returns the result or the empty
+// string if there is no such token.
+func consume(s string, c byte) string {
+ if s == "" || s[0] != c {
+ return ""
+ }
+ return strings.TrimSpace(s[1:])
+}
+
+func split(s string, c byte) (head, tail string) {
+ if i := strings.IndexByte(s, c); i >= 0 {
+ return strings.TrimSpace(s[:i]), strings.TrimSpace(s[i+1:])
+ }
+ return strings.TrimSpace(s), ""
+}
+
+// Add hack mapping to deal with a small number of cases that that occur
+// in Accept-Language (with reasonable frequency).
+var acceptFallback = map[string]language.Language{
+ "english": _en,
+ "deutsch": _de,
+ "italian": _it,
+ "french": _fr,
+ "*": _mul, // defined in the spec to match all languages.
+}
+
+type tagSort struct {
+ tag []Tag
+ q []float32
+}
+
+func (s *tagSort) Len() int {
+ return len(s.q)
+}
+
+func (s *tagSort) Less(i, j int) bool {
+ return s.q[i] > s.q[j]
+}
+
+func (s *tagSort) Swap(i, j int) {
+ s.tag[i], s.tag[j] = s.tag[j], s.tag[i]
+ s.q[i], s.q[j] = s.q[j], s.q[i]
+}
diff --git a/internal/language/compact/parse_test.go b/internal/language/compact/parse_test.go
new file mode 100644
index 0000000..2ff28bf
--- /dev/null
+++ b/internal/language/compact/parse_test.go
@@ -0,0 +1,390 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+import (
+ "strings"
+ "testing"
+
+ "golang.org/x/text/internal/language"
+)
+
+// equalTags compares language, script and region subtags only.
+func (t Tag) equalTags(a Tag) bool {
+ return t.lang() == a.lang() &&
+ t.script() == a.script() &&
+ t.region() == a.region()
+}
+
+var errSyntax = language.ErrSyntax
+
+type parseTest struct {
+ i int // the index of this test
+ in string
+ lang, script, region string
+ variants, ext string
+ extList []string // only used when more than one extension is present
+ invalid bool
+ rewrite bool // special rewrite not handled by parseTag
+ changed bool // string needed to be reformatted
+}
+
+func parseTests() []parseTest {
+ tests := []parseTest{
+ {in: "root", lang: "und"},
+ {in: "und", lang: "und"},
+ {in: "en", lang: "en"},
+
+ {in: "en-US-u-va-posix", lang: "en", region: "US", ext: "u-va-posix"},
+ {in: "ca-ES-valencia", lang: "ca", region: "ES", variants: "valencia"},
+ {in: "en-US-u-rg-gbzzzz", lang: "en", region: "US", ext: "u-rg-gbzzzz"},
+
+ {in: "xy", lang: "und", invalid: true},
+ {in: "en-ZY", lang: "en", invalid: true},
+ {in: "gsw", lang: "gsw"},
+ {in: "sr_Latn", lang: "sr", script: "Latn"},
+ {in: "af-Arab", lang: "af", script: "Arab"},
+ {in: "nl-BE", lang: "nl", region: "BE"},
+ {in: "es-419", lang: "es", region: "419"},
+ {in: "und-001", lang: "und", region: "001"},
+ {in: "de-latn-be", lang: "de", script: "Latn", region: "BE"},
+ // Variants
+ {in: "de-1901", lang: "de", variants: "1901"},
+ // Accept with unsuppressed script.
+ {in: "de-Latn-1901", lang: "de", script: "Latn", variants: "1901"},
+ // Specialized.
+ {in: "sl-rozaj", lang: "sl", variants: "rozaj"},
+ {in: "sl-rozaj-lipaw", lang: "sl", variants: "rozaj-lipaw"},
+ {in: "sl-rozaj-biske", lang: "sl", variants: "rozaj-biske"},
+ {in: "sl-rozaj-biske-1994", lang: "sl", variants: "rozaj-biske-1994"},
+ {in: "sl-rozaj-1994", lang: "sl", variants: "rozaj-1994"},
+ // Maximum number of variants while adhering to prefix rules.
+ {in: "sl-rozaj-biske-1994-alalc97-fonipa-fonupa-fonxsamp", lang: "sl", variants: "rozaj-biske-1994-alalc97-fonipa-fonupa-fonxsamp"},
+
+ // Sorting.
+ {in: "sl-1994-biske-rozaj", lang: "sl", variants: "rozaj-biske-1994", changed: true},
+ {in: "sl-rozaj-biske-1994-alalc97-fonupa-fonipa-fonxsamp", lang: "sl", variants: "rozaj-biske-1994-alalc97-fonipa-fonupa-fonxsamp", changed: true},
+ {in: "nl-fonxsamp-alalc97-fonipa-fonupa", lang: "nl", variants: "alalc97-fonipa-fonupa-fonxsamp", changed: true},
+
+ // Duplicates variants are removed, but not an error.
+ {in: "nl-fonupa-fonupa", lang: "nl", variants: "fonupa"},
+
+ // Variants that do not have correct prefixes. We still accept these.
+ {in: "de-Cyrl-1901", lang: "de", script: "Cyrl", variants: "1901"},
+ {in: "sl-rozaj-lipaw-1994", lang: "sl", variants: "rozaj-lipaw-1994"},
+ {in: "sl-1994-biske-rozaj-1994-biske-rozaj", lang: "sl", variants: "rozaj-biske-1994", changed: true},
+ {in: "de-Cyrl-1901", lang: "de", script: "Cyrl", variants: "1901"},
+
+ // Invalid variant.
+ {in: "de-1902", lang: "de", variants: "", invalid: true},
+
+ {in: "EN_CYRL", lang: "en", script: "Cyrl"},
+ // private use and extensions
+ {in: "x-a-b-c-d", ext: "x-a-b-c-d"},
+ {in: "x_A.-B-C_D", ext: "x-b-c-d", invalid: true, changed: true},
+ {in: "x-aa-bbbb-cccccccc-d", ext: "x-aa-bbbb-cccccccc-d"},
+ {in: "en-c_cc-b-bbb-a-aaa", lang: "en", changed: true, extList: []string{"a-aaa", "b-bbb", "c-cc"}},
+ {in: "en-x_cc-b-bbb-a-aaa", lang: "en", ext: "x-cc-b-bbb-a-aaa", changed: true},
+ {in: "en-c_cc-b-bbb-a-aaa-x-x", lang: "en", changed: true, extList: []string{"a-aaa", "b-bbb", "c-cc", "x-x"}},
+ {in: "en-v-c", lang: "en", ext: "", invalid: true},
+ {in: "en-v-abcdefghi", lang: "en", ext: "", invalid: true},
+ {in: "en-v-abc-x", lang: "en", ext: "v-abc", invalid: true},
+ {in: "en-v-abc-x-", lang: "en", ext: "v-abc", invalid: true},
+ {in: "en-v-abc-w-x-xx", lang: "en", extList: []string{"v-abc", "x-xx"}, invalid: true, changed: true},
+ {in: "en-v-abc-w-y-yx", lang: "en", extList: []string{"v-abc", "y-yx"}, invalid: true, changed: true},
+ {in: "en-v-c-abc", lang: "en", ext: "c-abc", invalid: true, changed: true},
+ {in: "en-v-w-abc", lang: "en", ext: "w-abc", invalid: true, changed: true},
+ {in: "en-v-x-abc", lang: "en", ext: "x-abc", invalid: true, changed: true},
+ {in: "en-v-x-a", lang: "en", ext: "x-a", invalid: true, changed: true},
+ {in: "en-9-aa-0-aa-z-bb-x-a", lang: "en", extList: []string{"0-aa", "9-aa", "z-bb", "x-a"}, changed: true},
+ {in: "en-u-c", lang: "en", ext: "", invalid: true},
+ {in: "en-u-co-phonebk", lang: "en", ext: "u-co-phonebk"},
+ {in: "en-u-co-phonebk-ca", lang: "en", ext: "u-co-phonebk", invalid: true},
+ {in: "en-u-nu-arabic-co-phonebk-ca", lang: "en", ext: "u-co-phonebk-nu-arabic", invalid: true, changed: true},
+ {in: "en-u-nu-arabic-co-phonebk-ca-x", lang: "en", ext: "u-co-phonebk-nu-arabic", invalid: true, changed: true},
+ {in: "en-u-nu-arabic-co-phonebk-ca-s", lang: "en", ext: "u-co-phonebk-nu-arabic", invalid: true, changed: true},
+ {in: "en-u-nu-arabic-co-phonebk-ca-a12345678", lang: "en", ext: "u-co-phonebk-nu-arabic", invalid: true, changed: true},
+ {in: "en-u-co-phonebook", lang: "en", ext: "", invalid: true},
+ {in: "en-u-co-phonebook-cu-xau", lang: "en", ext: "u-cu-xau", invalid: true, changed: true},
+ {in: "en-Cyrl-u-co-phonebk", lang: "en", script: "Cyrl", ext: "u-co-phonebk"},
+ {in: "en-US-u-co-phonebk", lang: "en", region: "US", ext: "u-co-phonebk"},
+ {in: "en-US-u-co-phonebk-cu-xau", lang: "en", region: "US", ext: "u-co-phonebk-cu-xau"},
+ {in: "en-scotland-u-co-phonebk", lang: "en", variants: "scotland", ext: "u-co-phonebk"},
+ {in: "en-u-cu-xua-co-phonebk", lang: "en", ext: "u-co-phonebk-cu-xua", changed: true},
+ {in: "en-u-def-abc-cu-xua-co-phonebk", lang: "en", ext: "u-abc-def-co-phonebk-cu-xua", changed: true},
+ {in: "en-u-def-abc", lang: "en", ext: "u-abc-def", changed: true},
+ {in: "en-u-cu-xua-co-phonebk-a-cd", lang: "en", extList: []string{"a-cd", "u-co-phonebk-cu-xua"}, changed: true},
+ // Invalid "u" extension. Drop invalid parts.
+ {in: "en-u-cu-co-phonebk", lang: "en", extList: []string{"u-co-phonebk"}, invalid: true, changed: true},
+ {in: "en-u-cu-xau-co", lang: "en", extList: []string{"u-cu-xau"}, invalid: true},
+ // We allow duplicate keys as the LDML spec does not explicitly prohibit it.
+ // TODO: Consider eliminating duplicates and returning an error.
+ {in: "en-u-cu-xau-co-phonebk-cu-xau", lang: "en", ext: "u-co-phonebk-cu-xau", changed: true},
+ {in: "en-t-en-Cyrl-NL-fonipa", lang: "en", ext: "t-en-cyrl-nl-fonipa", changed: true},
+ {in: "en-t-en-Cyrl-NL-fonipa-t0-abc-def", lang: "en", ext: "t-en-cyrl-nl-fonipa-t0-abc-def", changed: true},
+ {in: "en-t-t0-abcd", lang: "en", ext: "t-t0-abcd"},
+ // Not necessary to have changed here.
+ {in: "en-t-nl-abcd", lang: "en", ext: "t-nl", invalid: true},
+ {in: "en-t-nl-latn", lang: "en", ext: "t-nl-latn"},
+ {in: "en-t-t0-abcd-x-a", lang: "en", extList: []string{"t-t0-abcd", "x-a"}},
+ // invalid
+ {in: "", lang: "und", invalid: true},
+ {in: "-", lang: "und", invalid: true},
+ {in: "x", lang: "und", invalid: true},
+ {in: "x-", lang: "und", invalid: true},
+ {in: "x--", lang: "und", invalid: true},
+ {in: "a-a-b-c-d", lang: "und", invalid: true},
+ {in: "en-", lang: "en", invalid: true},
+ {in: "enne-", lang: "und", invalid: true},
+ {in: "en.", lang: "und", invalid: true},
+ {in: "en.-latn", lang: "und", invalid: true},
+ {in: "en.-en", lang: "en", invalid: true},
+ {in: "x-a-tooManyChars-c-d", ext: "x-a-c-d", invalid: true, changed: true},
+ {in: "a-tooManyChars-c-d", lang: "und", invalid: true},
+ // TODO: check key-value validity
+ // { in: "en-u-cu-xd", lang: "en", ext: "u-cu-xd", invalid: true },
+ {in: "en-t-abcd", lang: "en", invalid: true},
+ {in: "en-Latn-US-en", lang: "en", script: "Latn", region: "US", invalid: true},
+ // rewrites (more tests in TestGrandfathered)
+ {in: "zh-min-nan", lang: "nan"},
+ {in: "zh-yue", lang: "yue"},
+ {in: "zh-xiang", lang: "hsn", rewrite: true},
+ {in: "zh-guoyu", lang: "cmn", rewrite: true},
+ {in: "iw", lang: "iw"},
+ {in: "sgn-BE-FR", lang: "sfb", rewrite: true},
+ {in: "i-klingon", lang: "tlh", rewrite: true},
+ }
+ for i, tt := range tests {
+ tests[i].i = i
+ if tt.extList != nil {
+ tests[i].ext = strings.Join(tt.extList, "-")
+ }
+ if tt.ext != "" && tt.extList == nil {
+ tests[i].extList = []string{tt.ext}
+ }
+ }
+ return tests
+}
+
+// partChecks runs checks for each part by calling the function returned by f.
+func partChecks(t *testing.T, f func(*parseTest) (Tag, bool)) {
+ for i, tt := range parseTests() {
+ tag, skip := f(&tt)
+ if skip {
+ continue
+ }
+ if l, _ := language.ParseBase(tt.lang); l != tag.lang() {
+ t.Errorf("%d: lang was %q; want %q", i, tag.lang(), l)
+ }
+ if sc, _ := language.ParseScript(tt.script); sc != tag.script() {
+ t.Errorf("%d: script was %q; want %q", i, tag.script(), sc)
+ }
+ if r, _ := language.ParseRegion(tt.region); r != tag.region() {
+ t.Errorf("%d: region was %q; want %q", i, tag.region(), r)
+ }
+ v := tag.tag().Variants()
+ if v != "" {
+ v = v[1:]
+ }
+ if v != tt.variants {
+ t.Errorf("%d: variants was %q; want %q", i, v, tt.variants)
+ }
+ if e := strings.Join(tag.tag().Extensions(), "-"); e != tt.ext {
+ t.Errorf("%d: extensions were %q; want %q", i, e, tt.ext)
+ }
+ }
+}
+
+func TestParse(t *testing.T) {
+ partChecks(t, func(tt *parseTest) (id Tag, skip bool) {
+ id, _ = Raw.Parse(tt.in)
+ return id, false
+ })
+}
+
+func TestErrors(t *testing.T) {
+ mkInvalid := func(s string) error {
+ return language.NewValueError([]byte(s))
+ }
+ tests := []struct {
+ in string
+ out error
+ }{
+ // invalid subtags.
+ {"ac", mkInvalid("ac")},
+ {"AC", mkInvalid("ac")},
+ {"aa-Uuuu", mkInvalid("Uuuu")},
+ {"aa-AB", mkInvalid("AB")},
+ // ill-formed wins over invalid.
+ {"ac-u", errSyntax},
+ {"ac-u-ca", errSyntax},
+ {"ac-u-ca-co-pinyin", errSyntax},
+ {"noob", errSyntax},
+ }
+ for _, tt := range tests {
+ _, err := Parse(tt.in)
+ if err != tt.out {
+ t.Errorf("%s: was %q; want %q", tt.in, err, tt.out)
+ }
+ }
+}
+
+func TestCompose1(t *testing.T) {
+ partChecks(t, func(tt *parseTest) (id Tag, skip bool) {
+ l, _ := ParseBase(tt.lang)
+ s, _ := ParseScript(tt.script)
+ r, _ := ParseRegion(tt.region)
+ v := []Variant{}
+ for _, x := range strings.Split(tt.variants, "-") {
+ p, _ := ParseVariant(x)
+ v = append(v, p)
+ }
+ e := []Extension{}
+ for _, x := range tt.extList {
+ p, _ := ParseExtension(x)
+ e = append(e, p)
+ }
+ id, _ = Raw.Compose(l, s, r, v, e)
+ return id, false
+ })
+}
+
+func TestCompose2(t *testing.T) {
+ partChecks(t, func(tt *parseTest) (id Tag, skip bool) {
+ l, _ := ParseBase(tt.lang)
+ s, _ := ParseScript(tt.script)
+ r, _ := ParseRegion(tt.region)
+ p := []interface{}{l, s, r, s, r, l}
+ for _, x := range strings.Split(tt.variants, "-") {
+ if x != "" {
+ v, _ := ParseVariant(x)
+ p = append(p, v)
+ }
+ }
+ for _, x := range tt.extList {
+ e, _ := ParseExtension(x)
+ p = append(p, e)
+ }
+ id, _ = Raw.Compose(p...)
+ return id, false
+ })
+}
+
+func TestCompose3(t *testing.T) {
+ partChecks(t, func(tt *parseTest) (id Tag, skip bool) {
+ id, _ = Raw.Parse(tt.in)
+ id, _ = Raw.Compose(id)
+ return id, false
+ })
+}
+
+func mk(s string) Tag {
+ return Raw.Make(s)
+}
+
+func TestParseAcceptLanguage(t *testing.T) {
+ type res struct {
+ t Tag
+ q float32
+ }
+ en := []res{{mk("en"), 1.0}}
+ tests := []struct {
+ out []res
+ in string
+ ok bool
+ }{
+ {en, "en", true},
+ {en, " en", true},
+ {en, "en ", true},
+ {en, " en ", true},
+ {en, "en,", true},
+ {en, ",en", true},
+ {en, ",,,en,,,", true},
+ {en, ",en;q=1", true},
+
+ // We allow an empty input, contrary to spec.
+ {nil, "", true},
+ {[]res{{mk("aa"), 1}}, "aa;", true}, // allow unspecified weight
+
+ // errors
+ {nil, ";", false},
+ {nil, "$", false},
+ {nil, "e;", false},
+ {nil, "x;", false},
+ {nil, "x", false},
+ {nil, "ac", false}, // non-existing language
+ {nil, "aa;q", false},
+ {nil, "aa;q=", false},
+ {nil, "aa;q=.", false},
+
+ // odd fallbacks
+ {
+ []res{{mk("en"), 0.1}},
+ " english ;q=.1",
+ true,
+ },
+ {
+ []res{{mk("it"), 1.0}, {mk("de"), 1.0}, {mk("fr"), 1.0}},
+ " italian, deutsch, french",
+ true,
+ },
+
+ // lists
+ {
+ []res{{mk("en"), 0.1}},
+ "en;q=.1",
+ true,
+ },
+ {
+ []res{{mk("mul"), 1.0}},
+ "*",
+ true,
+ },
+ {
+ []res{{mk("en"), 1.0}, {mk("de"), 1.0}},
+ "en,de",
+ true,
+ },
+ {
+ []res{{mk("en"), 1.0}, {mk("de"), .5}},
+ "en,de;q=0.5",
+ true,
+ },
+ {
+ []res{{mk("de"), 0.8}, {mk("en"), 0.5}},
+ " en ; q = 0.5 , , de;q=0.8",
+ true,
+ },
+ {
+ []res{{mk("en"), 1.0}, {mk("de"), 1.0}, {mk("fr"), 1.0}, {mk("tlh"), 1.0}},
+ "en,de,fr,i-klingon",
+ true,
+ },
+ // sorting
+ {
+ []res{{mk("tlh"), 0.4}, {mk("de"), 0.2}, {mk("fr"), 0.2}, {mk("en"), 0.1}},
+ "en;q=0.1,de;q=0.2,fr;q=0.2,i-klingon;q=0.4",
+ true,
+ },
+ // dropping
+ {
+ []res{{mk("fr"), 0.2}, {mk("en"), 0.1}},
+ "en;q=0.1,de;q=0,fr;q=0.2,i-klingon;q=0.0",
+ true,
+ },
+ }
+ for i, tt := range tests {
+ tags, qs, e := ParseAcceptLanguage(tt.in)
+ if e == nil != tt.ok {
+ t.Errorf("%d:%s:err: was %v; want %v", i, tt.in, e == nil, tt.ok)
+ }
+ for j, tag := range tags {
+ if out := tt.out[j]; !tag.equalTags(out.t) || qs[j] != out.q {
+ t.Errorf("%d:%s: was %s, %1f; want %s, %1f", i, tt.in, tag, qs[j], out.t, out.q)
+ break
+ }
+ }
+ }
+}
diff --git a/internal/language/compact/tables.go b/internal/language/compact/tables.go
new file mode 100644
index 0000000..1825af6
--- /dev/null
+++ b/internal/language/compact/tables.go
@@ -0,0 +1,1305 @@
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+package language
+
+import "golang.org/x/text/internal/language"
+
+// CLDRVersion is the CLDR version from which the tables in this package are derived.
+const CLDRVersion = "32"
+
+const (
+ _de = 269
+ _en = 313
+ _fr = 350
+ _it = 505
+ _mo = 784
+ _no = 879
+ _nb = 839
+ _pt = 960
+ _sh = 1031
+ _mul = 806
+ _und = 0
+)
+const (
+ _001 = 1
+ _419 = 31
+ _BR = 65
+ _CA = 73
+ _ES = 110
+ _GB = 123
+ _MD = 188
+ _PT = 238
+ _UK = 306
+ _US = 309
+ _ZZ = 357
+ _XA = 323
+ _XC = 325
+ _XK = 333
+)
+const (
+ _Latn = 87
+ _Hani = 54
+ _Hans = 56
+ _Hant = 57
+ _Qaaa = 139
+ _Qaai = 147
+ _Qabx = 188
+ _Zinh = 236
+ _Zyyy = 241
+ _Zzzz = 242
+)
+
+// NumCompactTags is the number of common tags. The maximum tag is
+// NumCompactTags-1.
+const NumCompactTags = 775
+const (
+ undIndex compactID = 0
+ afIndex compactID = 1
+ afNAIndex compactID = 2
+ afZAIndex compactID = 3
+ agqIndex compactID = 4
+ agqCMIndex compactID = 5
+ akIndex compactID = 6
+ akGHIndex compactID = 7
+ amIndex compactID = 8
+ amETIndex compactID = 9
+ arIndex compactID = 10
+ ar001Index compactID = 11
+ arAEIndex compactID = 12
+ arBHIndex compactID = 13
+ arDJIndex compactID = 14
+ arDZIndex compactID = 15
+ arEGIndex compactID = 16
+ arEHIndex compactID = 17
+ arERIndex compactID = 18
+ arILIndex compactID = 19
+ arIQIndex compactID = 20
+ arJOIndex compactID = 21
+ arKMIndex compactID = 22
+ arKWIndex compactID = 23
+ arLBIndex compactID = 24
+ arLYIndex compactID = 25
+ arMAIndex compactID = 26
+ arMRIndex compactID = 27
+ arOMIndex compactID = 28
+ arPSIndex compactID = 29
+ arQAIndex compactID = 30
+ arSAIndex compactID = 31
+ arSDIndex compactID = 32
+ arSOIndex compactID = 33
+ arSSIndex compactID = 34
+ arSYIndex compactID = 35
+ arTDIndex compactID = 36
+ arTNIndex compactID = 37
+ arYEIndex compactID = 38
+ arsIndex compactID = 39
+ asIndex compactID = 40
+ asINIndex compactID = 41
+ asaIndex compactID = 42
+ asaTZIndex compactID = 43
+ astIndex compactID = 44
+ astESIndex compactID = 45
+ azIndex compactID = 46
+ azCyrlIndex compactID = 47
+ azCyrlAZIndex compactID = 48
+ azLatnIndex compactID = 49
+ azLatnAZIndex compactID = 50
+ basIndex compactID = 51
+ basCMIndex compactID = 52
+ beIndex compactID = 53
+ beBYIndex compactID = 54
+ bemIndex compactID = 55
+ bemZMIndex compactID = 56
+ bezIndex compactID = 57
+ bezTZIndex compactID = 58
+ bgIndex compactID = 59
+ bgBGIndex compactID = 60
+ bhIndex compactID = 61
+ bmIndex compactID = 62
+ bmMLIndex compactID = 63
+ bnIndex compactID = 64
+ bnBDIndex compactID = 65
+ bnINIndex compactID = 66
+ boIndex compactID = 67
+ boCNIndex compactID = 68
+ boINIndex compactID = 69
+ brIndex compactID = 70
+ brFRIndex compactID = 71
+ brxIndex compactID = 72
+ brxINIndex compactID = 73
+ bsIndex compactID = 74
+ bsCyrlIndex compactID = 75
+ bsCyrlBAIndex compactID = 76
+ bsLatnIndex compactID = 77
+ bsLatnBAIndex compactID = 78
+ caIndex compactID = 79
+ caADIndex compactID = 80
+ caESIndex compactID = 81
+ caFRIndex compactID = 82
+ caITIndex compactID = 83
+ ccpIndex compactID = 84
+ ccpBDIndex compactID = 85
+ ccpINIndex compactID = 86
+ ceIndex compactID = 87
+ ceRUIndex compactID = 88
+ cggIndex compactID = 89
+ cggUGIndex compactID = 90
+ chrIndex compactID = 91
+ chrUSIndex compactID = 92
+ ckbIndex compactID = 93
+ ckbIQIndex compactID = 94
+ ckbIRIndex compactID = 95
+ csIndex compactID = 96
+ csCZIndex compactID = 97
+ cuIndex compactID = 98
+ cuRUIndex compactID = 99
+ cyIndex compactID = 100
+ cyGBIndex compactID = 101
+ daIndex compactID = 102
+ daDKIndex compactID = 103
+ daGLIndex compactID = 104
+ davIndex compactID = 105
+ davKEIndex compactID = 106
+ deIndex compactID = 107
+ deATIndex compactID = 108
+ deBEIndex compactID = 109
+ deCHIndex compactID = 110
+ deDEIndex compactID = 111
+ deITIndex compactID = 112
+ deLIIndex compactID = 113
+ deLUIndex compactID = 114
+ djeIndex compactID = 115
+ djeNEIndex compactID = 116
+ dsbIndex compactID = 117
+ dsbDEIndex compactID = 118
+ duaIndex compactID = 119
+ duaCMIndex compactID = 120
+ dvIndex compactID = 121
+ dyoIndex compactID = 122
+ dyoSNIndex compactID = 123
+ dzIndex compactID = 124
+ dzBTIndex compactID = 125
+ ebuIndex compactID = 126
+ ebuKEIndex compactID = 127
+ eeIndex compactID = 128
+ eeGHIndex compactID = 129
+ eeTGIndex compactID = 130
+ elIndex compactID = 131
+ elCYIndex compactID = 132
+ elGRIndex compactID = 133
+ enIndex compactID = 134
+ en001Index compactID = 135
+ en150Index compactID = 136
+ enAGIndex compactID = 137
+ enAIIndex compactID = 138
+ enASIndex compactID = 139
+ enATIndex compactID = 140
+ enAUIndex compactID = 141
+ enBBIndex compactID = 142
+ enBEIndex compactID = 143
+ enBIIndex compactID = 144
+ enBMIndex compactID = 145
+ enBSIndex compactID = 146
+ enBWIndex compactID = 147
+ enBZIndex compactID = 148
+ enCAIndex compactID = 149
+ enCCIndex compactID = 150
+ enCHIndex compactID = 151
+ enCKIndex compactID = 152
+ enCMIndex compactID = 153
+ enCXIndex compactID = 154
+ enCYIndex compactID = 155
+ enDEIndex compactID = 156
+ enDGIndex compactID = 157
+ enDKIndex compactID = 158
+ enDMIndex compactID = 159
+ enERIndex compactID = 160
+ enFIIndex compactID = 161
+ enFJIndex compactID = 162
+ enFKIndex compactID = 163
+ enFMIndex compactID = 164
+ enGBIndex compactID = 165
+ enGDIndex compactID = 166
+ enGGIndex compactID = 167
+ enGHIndex compactID = 168
+ enGIIndex compactID = 169
+ enGMIndex compactID = 170
+ enGUIndex compactID = 171
+ enGYIndex compactID = 172
+ enHKIndex compactID = 173
+ enIEIndex compactID = 174
+ enILIndex compactID = 175
+ enIMIndex compactID = 176
+ enINIndex compactID = 177
+ enIOIndex compactID = 178
+ enJEIndex compactID = 179
+ enJMIndex compactID = 180
+ enKEIndex compactID = 181
+ enKIIndex compactID = 182
+ enKNIndex compactID = 183
+ enKYIndex compactID = 184
+ enLCIndex compactID = 185
+ enLRIndex compactID = 186
+ enLSIndex compactID = 187
+ enMGIndex compactID = 188
+ enMHIndex compactID = 189
+ enMOIndex compactID = 190
+ enMPIndex compactID = 191
+ enMSIndex compactID = 192
+ enMTIndex compactID = 193
+ enMUIndex compactID = 194
+ enMWIndex compactID = 195
+ enMYIndex compactID = 196
+ enNAIndex compactID = 197
+ enNFIndex compactID = 198
+ enNGIndex compactID = 199
+ enNLIndex compactID = 200
+ enNRIndex compactID = 201
+ enNUIndex compactID = 202
+ enNZIndex compactID = 203
+ enPGIndex compactID = 204
+ enPHIndex compactID = 205
+ enPKIndex compactID = 206
+ enPNIndex compactID = 207
+ enPRIndex compactID = 208
+ enPWIndex compactID = 209
+ enRWIndex compactID = 210
+ enSBIndex compactID = 211
+ enSCIndex compactID = 212
+ enSDIndex compactID = 213
+ enSEIndex compactID = 214
+ enSGIndex compactID = 215
+ enSHIndex compactID = 216
+ enSIIndex compactID = 217
+ enSLIndex compactID = 218
+ enSSIndex compactID = 219
+ enSXIndex compactID = 220
+ enSZIndex compactID = 221
+ enTCIndex compactID = 222
+ enTKIndex compactID = 223
+ enTOIndex compactID = 224
+ enTTIndex compactID = 225
+ enTVIndex compactID = 226
+ enTZIndex compactID = 227
+ enUGIndex compactID = 228
+ enUMIndex compactID = 229
+ enUSIndex compactID = 230
+ enVCIndex compactID = 231
+ enVGIndex compactID = 232
+ enVIIndex compactID = 233
+ enVUIndex compactID = 234
+ enWSIndex compactID = 235
+ enZAIndex compactID = 236
+ enZMIndex compactID = 237
+ enZWIndex compactID = 238
+ eoIndex compactID = 239
+ eo001Index compactID = 240
+ esIndex compactID = 241
+ es419Index compactID = 242
+ esARIndex compactID = 243
+ esBOIndex compactID = 244
+ esBRIndex compactID = 245
+ esBZIndex compactID = 246
+ esCLIndex compactID = 247
+ esCOIndex compactID = 248
+ esCRIndex compactID = 249
+ esCUIndex compactID = 250
+ esDOIndex compactID = 251
+ esEAIndex compactID = 252
+ esECIndex compactID = 253
+ esESIndex compactID = 254
+ esGQIndex compactID = 255
+ esGTIndex compactID = 256
+ esHNIndex compactID = 257
+ esICIndex compactID = 258
+ esMXIndex compactID = 259
+ esNIIndex compactID = 260
+ esPAIndex compactID = 261
+ esPEIndex compactID = 262
+ esPHIndex compactID = 263
+ esPRIndex compactID = 264
+ esPYIndex compactID = 265
+ esSVIndex compactID = 266
+ esUSIndex compactID = 267
+ esUYIndex compactID = 268
+ esVEIndex compactID = 269
+ etIndex compactID = 270
+ etEEIndex compactID = 271
+ euIndex compactID = 272
+ euESIndex compactID = 273
+ ewoIndex compactID = 274
+ ewoCMIndex compactID = 275
+ faIndex compactID = 276
+ faAFIndex compactID = 277
+ faIRIndex compactID = 278
+ ffIndex compactID = 279
+ ffCMIndex compactID = 280
+ ffGNIndex compactID = 281
+ ffMRIndex compactID = 282
+ ffSNIndex compactID = 283
+ fiIndex compactID = 284
+ fiFIIndex compactID = 285
+ filIndex compactID = 286
+ filPHIndex compactID = 287
+ foIndex compactID = 288
+ foDKIndex compactID = 289
+ foFOIndex compactID = 290
+ frIndex compactID = 291
+ frBEIndex compactID = 292
+ frBFIndex compactID = 293
+ frBIIndex compactID = 294
+ frBJIndex compactID = 295
+ frBLIndex compactID = 296
+ frCAIndex compactID = 297
+ frCDIndex compactID = 298
+ frCFIndex compactID = 299
+ frCGIndex compactID = 300
+ frCHIndex compactID = 301
+ frCIIndex compactID = 302
+ frCMIndex compactID = 303
+ frDJIndex compactID = 304
+ frDZIndex compactID = 305
+ frFRIndex compactID = 306
+ frGAIndex compactID = 307
+ frGFIndex compactID = 308
+ frGNIndex compactID = 309
+ frGPIndex compactID = 310
+ frGQIndex compactID = 311
+ frHTIndex compactID = 312
+ frKMIndex compactID = 313
+ frLUIndex compactID = 314
+ frMAIndex compactID = 315
+ frMCIndex compactID = 316
+ frMFIndex compactID = 317
+ frMGIndex compactID = 318
+ frMLIndex compactID = 319
+ frMQIndex compactID = 320
+ frMRIndex compactID = 321
+ frMUIndex compactID = 322
+ frNCIndex compactID = 323
+ frNEIndex compactID = 324
+ frPFIndex compactID = 325
+ frPMIndex compactID = 326
+ frREIndex compactID = 327
+ frRWIndex compactID = 328
+ frSCIndex compactID = 329
+ frSNIndex compactID = 330
+ frSYIndex compactID = 331
+ frTDIndex compactID = 332
+ frTGIndex compactID = 333
+ frTNIndex compactID = 334
+ frVUIndex compactID = 335
+ frWFIndex compactID = 336
+ frYTIndex compactID = 337
+ furIndex compactID = 338
+ furITIndex compactID = 339
+ fyIndex compactID = 340
+ fyNLIndex compactID = 341
+ gaIndex compactID = 342
+ gaIEIndex compactID = 343
+ gdIndex compactID = 344
+ gdGBIndex compactID = 345
+ glIndex compactID = 346
+ glESIndex compactID = 347
+ gswIndex compactID = 348
+ gswCHIndex compactID = 349
+ gswFRIndex compactID = 350
+ gswLIIndex compactID = 351
+ guIndex compactID = 352
+ guINIndex compactID = 353
+ guwIndex compactID = 354
+ guzIndex compactID = 355
+ guzKEIndex compactID = 356
+ gvIndex compactID = 357
+ gvIMIndex compactID = 358
+ haIndex compactID = 359
+ haGHIndex compactID = 360
+ haNEIndex compactID = 361
+ haNGIndex compactID = 362
+ hawIndex compactID = 363
+ hawUSIndex compactID = 364
+ heIndex compactID = 365
+ heILIndex compactID = 366
+ hiIndex compactID = 367
+ hiINIndex compactID = 368
+ hrIndex compactID = 369
+ hrBAIndex compactID = 370
+ hrHRIndex compactID = 371
+ hsbIndex compactID = 372
+ hsbDEIndex compactID = 373
+ huIndex compactID = 374
+ huHUIndex compactID = 375
+ hyIndex compactID = 376
+ hyAMIndex compactID = 377
+ idIndex compactID = 378
+ idIDIndex compactID = 379
+ igIndex compactID = 380
+ igNGIndex compactID = 381
+ iiIndex compactID = 382
+ iiCNIndex compactID = 383
+ inIndex compactID = 384
+ ioIndex compactID = 385
+ isIndex compactID = 386
+ isISIndex compactID = 387
+ itIndex compactID = 388
+ itCHIndex compactID = 389
+ itITIndex compactID = 390
+ itSMIndex compactID = 391
+ itVAIndex compactID = 392
+ iuIndex compactID = 393
+ iwIndex compactID = 394
+ jaIndex compactID = 395
+ jaJPIndex compactID = 396
+ jboIndex compactID = 397
+ jgoIndex compactID = 398
+ jgoCMIndex compactID = 399
+ jiIndex compactID = 400
+ jmcIndex compactID = 401
+ jmcTZIndex compactID = 402
+ jvIndex compactID = 403
+ jwIndex compactID = 404
+ kaIndex compactID = 405
+ kaGEIndex compactID = 406
+ kabIndex compactID = 407
+ kabDZIndex compactID = 408
+ kajIndex compactID = 409
+ kamIndex compactID = 410
+ kamKEIndex compactID = 411
+ kcgIndex compactID = 412
+ kdeIndex compactID = 413
+ kdeTZIndex compactID = 414
+ keaIndex compactID = 415
+ keaCVIndex compactID = 416
+ khqIndex compactID = 417
+ khqMLIndex compactID = 418
+ kiIndex compactID = 419
+ kiKEIndex compactID = 420
+ kkIndex compactID = 421
+ kkKZIndex compactID = 422
+ kkjIndex compactID = 423
+ kkjCMIndex compactID = 424
+ klIndex compactID = 425
+ klGLIndex compactID = 426
+ klnIndex compactID = 427
+ klnKEIndex compactID = 428
+ kmIndex compactID = 429
+ kmKHIndex compactID = 430
+ knIndex compactID = 431
+ knINIndex compactID = 432
+ koIndex compactID = 433
+ koKPIndex compactID = 434
+ koKRIndex compactID = 435
+ kokIndex compactID = 436
+ kokINIndex compactID = 437
+ ksIndex compactID = 438
+ ksINIndex compactID = 439
+ ksbIndex compactID = 440
+ ksbTZIndex compactID = 441
+ ksfIndex compactID = 442
+ ksfCMIndex compactID = 443
+ kshIndex compactID = 444
+ kshDEIndex compactID = 445
+ kuIndex compactID = 446
+ kwIndex compactID = 447
+ kwGBIndex compactID = 448
+ kyIndex compactID = 449
+ kyKGIndex compactID = 450
+ lagIndex compactID = 451
+ lagTZIndex compactID = 452
+ lbIndex compactID = 453
+ lbLUIndex compactID = 454
+ lgIndex compactID = 455
+ lgUGIndex compactID = 456
+ lktIndex compactID = 457
+ lktUSIndex compactID = 458
+ lnIndex compactID = 459
+ lnAOIndex compactID = 460
+ lnCDIndex compactID = 461
+ lnCFIndex compactID = 462
+ lnCGIndex compactID = 463
+ loIndex compactID = 464
+ loLAIndex compactID = 465
+ lrcIndex compactID = 466
+ lrcIQIndex compactID = 467
+ lrcIRIndex compactID = 468
+ ltIndex compactID = 469
+ ltLTIndex compactID = 470
+ luIndex compactID = 471
+ luCDIndex compactID = 472
+ luoIndex compactID = 473
+ luoKEIndex compactID = 474
+ luyIndex compactID = 475
+ luyKEIndex compactID = 476
+ lvIndex compactID = 477
+ lvLVIndex compactID = 478
+ masIndex compactID = 479
+ masKEIndex compactID = 480
+ masTZIndex compactID = 481
+ merIndex compactID = 482
+ merKEIndex compactID = 483
+ mfeIndex compactID = 484
+ mfeMUIndex compactID = 485
+ mgIndex compactID = 486
+ mgMGIndex compactID = 487
+ mghIndex compactID = 488
+ mghMZIndex compactID = 489
+ mgoIndex compactID = 490
+ mgoCMIndex compactID = 491
+ mkIndex compactID = 492
+ mkMKIndex compactID = 493
+ mlIndex compactID = 494
+ mlINIndex compactID = 495
+ mnIndex compactID = 496
+ mnMNIndex compactID = 497
+ moIndex compactID = 498
+ mrIndex compactID = 499
+ mrINIndex compactID = 500
+ msIndex compactID = 501
+ msBNIndex compactID = 502
+ msMYIndex compactID = 503
+ msSGIndex compactID = 504
+ mtIndex compactID = 505
+ mtMTIndex compactID = 506
+ muaIndex compactID = 507
+ muaCMIndex compactID = 508
+ myIndex compactID = 509
+ myMMIndex compactID = 510
+ mznIndex compactID = 511
+ mznIRIndex compactID = 512
+ nahIndex compactID = 513
+ naqIndex compactID = 514
+ naqNAIndex compactID = 515
+ nbIndex compactID = 516
+ nbNOIndex compactID = 517
+ nbSJIndex compactID = 518
+ ndIndex compactID = 519
+ ndZWIndex compactID = 520
+ ndsIndex compactID = 521
+ ndsDEIndex compactID = 522
+ ndsNLIndex compactID = 523
+ neIndex compactID = 524
+ neINIndex compactID = 525
+ neNPIndex compactID = 526
+ nlIndex compactID = 527
+ nlAWIndex compactID = 528
+ nlBEIndex compactID = 529
+ nlBQIndex compactID = 530
+ nlCWIndex compactID = 531
+ nlNLIndex compactID = 532
+ nlSRIndex compactID = 533
+ nlSXIndex compactID = 534
+ nmgIndex compactID = 535
+ nmgCMIndex compactID = 536
+ nnIndex compactID = 537
+ nnNOIndex compactID = 538
+ nnhIndex compactID = 539
+ nnhCMIndex compactID = 540
+ noIndex compactID = 541
+ nqoIndex compactID = 542
+ nrIndex compactID = 543
+ nsoIndex compactID = 544
+ nusIndex compactID = 545
+ nusSSIndex compactID = 546
+ nyIndex compactID = 547
+ nynIndex compactID = 548
+ nynUGIndex compactID = 549
+ omIndex compactID = 550
+ omETIndex compactID = 551
+ omKEIndex compactID = 552
+ orIndex compactID = 553
+ orINIndex compactID = 554
+ osIndex compactID = 555
+ osGEIndex compactID = 556
+ osRUIndex compactID = 557
+ paIndex compactID = 558
+ paArabIndex compactID = 559
+ paArabPKIndex compactID = 560
+ paGuruIndex compactID = 561
+ paGuruINIndex compactID = 562
+ papIndex compactID = 563
+ plIndex compactID = 564
+ plPLIndex compactID = 565
+ prgIndex compactID = 566
+ prg001Index compactID = 567
+ psIndex compactID = 568
+ psAFIndex compactID = 569
+ ptIndex compactID = 570
+ ptAOIndex compactID = 571
+ ptBRIndex compactID = 572
+ ptCHIndex compactID = 573
+ ptCVIndex compactID = 574
+ ptGQIndex compactID = 575
+ ptGWIndex compactID = 576
+ ptLUIndex compactID = 577
+ ptMOIndex compactID = 578
+ ptMZIndex compactID = 579
+ ptPTIndex compactID = 580
+ ptSTIndex compactID = 581
+ ptTLIndex compactID = 582
+ quIndex compactID = 583
+ quBOIndex compactID = 584
+ quECIndex compactID = 585
+ quPEIndex compactID = 586
+ rmIndex compactID = 587
+ rmCHIndex compactID = 588
+ rnIndex compactID = 589
+ rnBIIndex compactID = 590
+ roIndex compactID = 591
+ roMDIndex compactID = 592
+ roROIndex compactID = 593
+ rofIndex compactID = 594
+ rofTZIndex compactID = 595
+ ruIndex compactID = 596
+ ruBYIndex compactID = 597
+ ruKGIndex compactID = 598
+ ruKZIndex compactID = 599
+ ruMDIndex compactID = 600
+ ruRUIndex compactID = 601
+ ruUAIndex compactID = 602
+ rwIndex compactID = 603
+ rwRWIndex compactID = 604
+ rwkIndex compactID = 605
+ rwkTZIndex compactID = 606
+ sahIndex compactID = 607
+ sahRUIndex compactID = 608
+ saqIndex compactID = 609
+ saqKEIndex compactID = 610
+ sbpIndex compactID = 611
+ sbpTZIndex compactID = 612
+ sdIndex compactID = 613
+ sdPKIndex compactID = 614
+ sdhIndex compactID = 615
+ seIndex compactID = 616
+ seFIIndex compactID = 617
+ seNOIndex compactID = 618
+ seSEIndex compactID = 619
+ sehIndex compactID = 620
+ sehMZIndex compactID = 621
+ sesIndex compactID = 622
+ sesMLIndex compactID = 623
+ sgIndex compactID = 624
+ sgCFIndex compactID = 625
+ shIndex compactID = 626
+ shiIndex compactID = 627
+ shiLatnIndex compactID = 628
+ shiLatnMAIndex compactID = 629
+ shiTfngIndex compactID = 630
+ shiTfngMAIndex compactID = 631
+ siIndex compactID = 632
+ siLKIndex compactID = 633
+ skIndex compactID = 634
+ skSKIndex compactID = 635
+ slIndex compactID = 636
+ slSIIndex compactID = 637
+ smaIndex compactID = 638
+ smiIndex compactID = 639
+ smjIndex compactID = 640
+ smnIndex compactID = 641
+ smnFIIndex compactID = 642
+ smsIndex compactID = 643
+ snIndex compactID = 644
+ snZWIndex compactID = 645
+ soIndex compactID = 646
+ soDJIndex compactID = 647
+ soETIndex compactID = 648
+ soKEIndex compactID = 649
+ soSOIndex compactID = 650
+ sqIndex compactID = 651
+ sqALIndex compactID = 652
+ sqMKIndex compactID = 653
+ sqXKIndex compactID = 654
+ srIndex compactID = 655
+ srCyrlIndex compactID = 656
+ srCyrlBAIndex compactID = 657
+ srCyrlMEIndex compactID = 658
+ srCyrlRSIndex compactID = 659
+ srCyrlXKIndex compactID = 660
+ srLatnIndex compactID = 661
+ srLatnBAIndex compactID = 662
+ srLatnMEIndex compactID = 663
+ srLatnRSIndex compactID = 664
+ srLatnXKIndex compactID = 665
+ ssIndex compactID = 666
+ ssyIndex compactID = 667
+ stIndex compactID = 668
+ svIndex compactID = 669
+ svAXIndex compactID = 670
+ svFIIndex compactID = 671
+ svSEIndex compactID = 672
+ swIndex compactID = 673
+ swCDIndex compactID = 674
+ swKEIndex compactID = 675
+ swTZIndex compactID = 676
+ swUGIndex compactID = 677
+ syrIndex compactID = 678
+ taIndex compactID = 679
+ taINIndex compactID = 680
+ taLKIndex compactID = 681
+ taMYIndex compactID = 682
+ taSGIndex compactID = 683
+ teIndex compactID = 684
+ teINIndex compactID = 685
+ teoIndex compactID = 686
+ teoKEIndex compactID = 687
+ teoUGIndex compactID = 688
+ tgIndex compactID = 689
+ tgTJIndex compactID = 690
+ thIndex compactID = 691
+ thTHIndex compactID = 692
+ tiIndex compactID = 693
+ tiERIndex compactID = 694
+ tiETIndex compactID = 695
+ tigIndex compactID = 696
+ tkIndex compactID = 697
+ tkTMIndex compactID = 698
+ tlIndex compactID = 699
+ tnIndex compactID = 700
+ toIndex compactID = 701
+ toTOIndex compactID = 702
+ trIndex compactID = 703
+ trCYIndex compactID = 704
+ trTRIndex compactID = 705
+ tsIndex compactID = 706
+ ttIndex compactID = 707
+ ttRUIndex compactID = 708
+ twqIndex compactID = 709
+ twqNEIndex compactID = 710
+ tzmIndex compactID = 711
+ tzmMAIndex compactID = 712
+ ugIndex compactID = 713
+ ugCNIndex compactID = 714
+ ukIndex compactID = 715
+ ukUAIndex compactID = 716
+ urIndex compactID = 717
+ urINIndex compactID = 718
+ urPKIndex compactID = 719
+ uzIndex compactID = 720
+ uzArabIndex compactID = 721
+ uzArabAFIndex compactID = 722
+ uzCyrlIndex compactID = 723
+ uzCyrlUZIndex compactID = 724
+ uzLatnIndex compactID = 725
+ uzLatnUZIndex compactID = 726
+ vaiIndex compactID = 727
+ vaiLatnIndex compactID = 728
+ vaiLatnLRIndex compactID = 729
+ vaiVaiiIndex compactID = 730
+ vaiVaiiLRIndex compactID = 731
+ veIndex compactID = 732
+ viIndex compactID = 733
+ viVNIndex compactID = 734
+ voIndex compactID = 735
+ vo001Index compactID = 736
+ vunIndex compactID = 737
+ vunTZIndex compactID = 738
+ waIndex compactID = 739
+ waeIndex compactID = 740
+ waeCHIndex compactID = 741
+ woIndex compactID = 742
+ woSNIndex compactID = 743
+ xhIndex compactID = 744
+ xogIndex compactID = 745
+ xogUGIndex compactID = 746
+ yavIndex compactID = 747
+ yavCMIndex compactID = 748
+ yiIndex compactID = 749
+ yi001Index compactID = 750
+ yoIndex compactID = 751
+ yoBJIndex compactID = 752
+ yoNGIndex compactID = 753
+ yueIndex compactID = 754
+ yueHansIndex compactID = 755
+ yueHansCNIndex compactID = 756
+ yueHantIndex compactID = 757
+ yueHantHKIndex compactID = 758
+ zghIndex compactID = 759
+ zghMAIndex compactID = 760
+ zhIndex compactID = 761
+ zhHansIndex compactID = 762
+ zhHansCNIndex compactID = 763
+ zhHansHKIndex compactID = 764
+ zhHansMOIndex compactID = 765
+ zhHansSGIndex compactID = 766
+ zhHantIndex compactID = 767
+ zhHantHKIndex compactID = 768
+ zhHantMOIndex compactID = 769
+ zhHantTWIndex compactID = 770
+ zuIndex compactID = 771
+ zuZAIndex compactID = 772
+ caESvalenciaIndex compactID = 773
+ enUSuvaposixIndex compactID = 774
+)
+
+var coreTags = []language.CompactCoreInfo{ // 773 elements
+ // Entry 0 - 1F
+ 0x00000000, 0x01600000, 0x016000d2, 0x01600161,
+ 0x01c00000, 0x01c00052, 0x02100000, 0x02100080,
+ 0x02700000, 0x0270006f, 0x03a00000, 0x03a00001,
+ 0x03a00023, 0x03a00039, 0x03a00062, 0x03a00067,
+ 0x03a0006b, 0x03a0006c, 0x03a0006d, 0x03a00097,
+ 0x03a0009b, 0x03a000a1, 0x03a000a8, 0x03a000ac,
+ 0x03a000b0, 0x03a000b9, 0x03a000ba, 0x03a000c9,
+ 0x03a000e1, 0x03a000ed, 0x03a000f3, 0x03a00108,
+ // Entry 20 - 3F
+ 0x03a0010b, 0x03a00115, 0x03a00117, 0x03a0011c,
+ 0x03a00120, 0x03a00128, 0x03a0015e, 0x04000000,
+ 0x04300000, 0x04300099, 0x04400000, 0x0440012f,
+ 0x04800000, 0x0480006e, 0x05800000, 0x0581f000,
+ 0x0581f032, 0x05857000, 0x05857032, 0x05e00000,
+ 0x05e00052, 0x07100000, 0x07100047, 0x07500000,
+ 0x07500162, 0x07900000, 0x0790012f, 0x07e00000,
+ 0x07e00038, 0x08200000, 0x0a000000, 0x0a0000c3,
+ // Entry 40 - 5F
+ 0x0a500000, 0x0a500035, 0x0a500099, 0x0a900000,
+ 0x0a900053, 0x0a900099, 0x0b200000, 0x0b200078,
+ 0x0b500000, 0x0b500099, 0x0b700000, 0x0b71f000,
+ 0x0b71f033, 0x0b757000, 0x0b757033, 0x0d700000,
+ 0x0d700022, 0x0d70006e, 0x0d700078, 0x0d70009e,
+ 0x0db00000, 0x0db00035, 0x0db00099, 0x0dc00000,
+ 0x0dc00106, 0x0df00000, 0x0df00131, 0x0e500000,
+ 0x0e500135, 0x0e900000, 0x0e90009b, 0x0e90009c,
+ // Entry 60 - 7F
+ 0x0fa00000, 0x0fa0005e, 0x0fe00000, 0x0fe00106,
+ 0x10000000, 0x1000007b, 0x10100000, 0x10100063,
+ 0x10100082, 0x10800000, 0x108000a4, 0x10d00000,
+ 0x10d0002e, 0x10d00036, 0x10d0004e, 0x10d00060,
+ 0x10d0009e, 0x10d000b2, 0x10d000b7, 0x11700000,
+ 0x117000d4, 0x11f00000, 0x11f00060, 0x12400000,
+ 0x12400052, 0x12800000, 0x12b00000, 0x12b00114,
+ 0x12d00000, 0x12d00043, 0x12f00000, 0x12f000a4,
+ // Entry 80 - 9F
+ 0x13000000, 0x13000080, 0x13000122, 0x13600000,
+ 0x1360005d, 0x13600087, 0x13900000, 0x13900001,
+ 0x1390001a, 0x13900025, 0x13900026, 0x1390002d,
+ 0x1390002e, 0x1390002f, 0x13900034, 0x13900036,
+ 0x1390003a, 0x1390003d, 0x13900042, 0x13900046,
+ 0x13900048, 0x13900049, 0x1390004a, 0x1390004e,
+ 0x13900050, 0x13900052, 0x1390005c, 0x1390005d,
+ 0x13900060, 0x13900061, 0x13900063, 0x13900064,
+ // Entry A0 - BF
+ 0x1390006d, 0x13900072, 0x13900073, 0x13900074,
+ 0x13900075, 0x1390007b, 0x1390007c, 0x1390007f,
+ 0x13900080, 0x13900081, 0x13900083, 0x1390008a,
+ 0x1390008c, 0x1390008d, 0x13900096, 0x13900097,
+ 0x13900098, 0x13900099, 0x1390009a, 0x1390009f,
+ 0x139000a0, 0x139000a4, 0x139000a7, 0x139000a9,
+ 0x139000ad, 0x139000b1, 0x139000b4, 0x139000b5,
+ 0x139000bf, 0x139000c0, 0x139000c6, 0x139000c7,
+ // Entry C0 - DF
+ 0x139000ca, 0x139000cb, 0x139000cc, 0x139000ce,
+ 0x139000d0, 0x139000d2, 0x139000d5, 0x139000d6,
+ 0x139000d9, 0x139000dd, 0x139000df, 0x139000e0,
+ 0x139000e6, 0x139000e7, 0x139000e8, 0x139000eb,
+ 0x139000ec, 0x139000f0, 0x13900107, 0x13900109,
+ 0x1390010a, 0x1390010b, 0x1390010c, 0x1390010d,
+ 0x1390010e, 0x1390010f, 0x13900112, 0x13900117,
+ 0x1390011b, 0x1390011d, 0x1390011f, 0x13900125,
+ // Entry E0 - FF
+ 0x13900129, 0x1390012c, 0x1390012d, 0x1390012f,
+ 0x13900131, 0x13900133, 0x13900135, 0x13900139,
+ 0x1390013c, 0x1390013d, 0x1390013f, 0x13900142,
+ 0x13900161, 0x13900162, 0x13900164, 0x13c00000,
+ 0x13c00001, 0x13e00000, 0x13e0001f, 0x13e0002c,
+ 0x13e0003f, 0x13e00041, 0x13e00048, 0x13e00051,
+ 0x13e00054, 0x13e00056, 0x13e00059, 0x13e00065,
+ 0x13e00068, 0x13e00069, 0x13e0006e, 0x13e00086,
+ // Entry 100 - 11F
+ 0x13e00089, 0x13e0008f, 0x13e00094, 0x13e000cf,
+ 0x13e000d8, 0x13e000e2, 0x13e000e4, 0x13e000e7,
+ 0x13e000ec, 0x13e000f1, 0x13e0011a, 0x13e00135,
+ 0x13e00136, 0x13e0013b, 0x14000000, 0x1400006a,
+ 0x14500000, 0x1450006e, 0x14600000, 0x14600052,
+ 0x14800000, 0x14800024, 0x1480009c, 0x14e00000,
+ 0x14e00052, 0x14e00084, 0x14e000c9, 0x14e00114,
+ 0x15100000, 0x15100072, 0x15300000, 0x153000e7,
+ // Entry 120 - 13F
+ 0x15800000, 0x15800063, 0x15800076, 0x15e00000,
+ 0x15e00036, 0x15e00037, 0x15e0003a, 0x15e0003b,
+ 0x15e0003c, 0x15e00049, 0x15e0004b, 0x15e0004c,
+ 0x15e0004d, 0x15e0004e, 0x15e0004f, 0x15e00052,
+ 0x15e00062, 0x15e00067, 0x15e00078, 0x15e0007a,
+ 0x15e0007e, 0x15e00084, 0x15e00085, 0x15e00086,
+ 0x15e00091, 0x15e000a8, 0x15e000b7, 0x15e000ba,
+ 0x15e000bb, 0x15e000be, 0x15e000bf, 0x15e000c3,
+ // Entry 140 - 15F
+ 0x15e000c8, 0x15e000c9, 0x15e000cc, 0x15e000d3,
+ 0x15e000d4, 0x15e000e5, 0x15e000ea, 0x15e00102,
+ 0x15e00107, 0x15e0010a, 0x15e00114, 0x15e0011c,
+ 0x15e00120, 0x15e00122, 0x15e00128, 0x15e0013f,
+ 0x15e00140, 0x15e0015f, 0x16900000, 0x1690009e,
+ 0x16d00000, 0x16d000d9, 0x16e00000, 0x16e00096,
+ 0x17e00000, 0x17e0007b, 0x19000000, 0x1900006e,
+ 0x1a300000, 0x1a30004e, 0x1a300078, 0x1a3000b2,
+ // Entry 160 - 17F
+ 0x1a400000, 0x1a400099, 0x1a900000, 0x1ab00000,
+ 0x1ab000a4, 0x1ac00000, 0x1ac00098, 0x1b400000,
+ 0x1b400080, 0x1b4000d4, 0x1b4000d6, 0x1b800000,
+ 0x1b800135, 0x1bc00000, 0x1bc00097, 0x1be00000,
+ 0x1be00099, 0x1d100000, 0x1d100033, 0x1d100090,
+ 0x1d200000, 0x1d200060, 0x1d500000, 0x1d500092,
+ 0x1d700000, 0x1d700028, 0x1e100000, 0x1e100095,
+ 0x1e700000, 0x1e7000d6, 0x1ea00000, 0x1ea00053,
+ // Entry 180 - 19F
+ 0x1f300000, 0x1f500000, 0x1f800000, 0x1f80009d,
+ 0x1f900000, 0x1f90004e, 0x1f90009e, 0x1f900113,
+ 0x1f900138, 0x1fa00000, 0x1fb00000, 0x20000000,
+ 0x200000a2, 0x20300000, 0x20700000, 0x20700052,
+ 0x20800000, 0x20a00000, 0x20a0012f, 0x20e00000,
+ 0x20f00000, 0x21000000, 0x2100007d, 0x21200000,
+ 0x21200067, 0x21600000, 0x21700000, 0x217000a4,
+ 0x21f00000, 0x22300000, 0x2230012f, 0x22700000,
+ // Entry 1A0 - 1BF
+ 0x2270005a, 0x23400000, 0x234000c3, 0x23900000,
+ 0x239000a4, 0x24200000, 0x242000ae, 0x24400000,
+ 0x24400052, 0x24500000, 0x24500082, 0x24600000,
+ 0x246000a4, 0x24a00000, 0x24a000a6, 0x25100000,
+ 0x25100099, 0x25400000, 0x254000aa, 0x254000ab,
+ 0x25600000, 0x25600099, 0x26a00000, 0x26a00099,
+ 0x26b00000, 0x26b0012f, 0x26d00000, 0x26d00052,
+ 0x26e00000, 0x26e00060, 0x27400000, 0x28100000,
+ // Entry 1C0 - 1DF
+ 0x2810007b, 0x28a00000, 0x28a000a5, 0x29100000,
+ 0x2910012f, 0x29500000, 0x295000b7, 0x2a300000,
+ 0x2a300131, 0x2af00000, 0x2af00135, 0x2b500000,
+ 0x2b50002a, 0x2b50004b, 0x2b50004c, 0x2b50004d,
+ 0x2b800000, 0x2b8000af, 0x2bf00000, 0x2bf0009b,
+ 0x2bf0009c, 0x2c000000, 0x2c0000b6, 0x2c200000,
+ 0x2c20004b, 0x2c400000, 0x2c4000a4, 0x2c500000,
+ 0x2c5000a4, 0x2c700000, 0x2c7000b8, 0x2d100000,
+ // Entry 1E0 - 1FF
+ 0x2d1000a4, 0x2d10012f, 0x2e900000, 0x2e9000a4,
+ 0x2ed00000, 0x2ed000cc, 0x2f100000, 0x2f1000bf,
+ 0x2f200000, 0x2f2000d1, 0x2f400000, 0x2f400052,
+ 0x2ff00000, 0x2ff000c2, 0x30400000, 0x30400099,
+ 0x30b00000, 0x30b000c5, 0x31000000, 0x31b00000,
+ 0x31b00099, 0x31f00000, 0x31f0003e, 0x31f000d0,
+ 0x31f0010d, 0x32000000, 0x320000cb, 0x32500000,
+ 0x32500052, 0x33100000, 0x331000c4, 0x33a00000,
+ // Entry 200 - 21F
+ 0x33a0009c, 0x34100000, 0x34500000, 0x345000d2,
+ 0x34700000, 0x347000da, 0x34700110, 0x34e00000,
+ 0x34e00164, 0x35000000, 0x35000060, 0x350000d9,
+ 0x35100000, 0x35100099, 0x351000db, 0x36700000,
+ 0x36700030, 0x36700036, 0x36700040, 0x3670005b,
+ 0x367000d9, 0x36700116, 0x3670011b, 0x36800000,
+ 0x36800052, 0x36a00000, 0x36a000da, 0x36c00000,
+ 0x36c00052, 0x36f00000, 0x37500000, 0x37600000,
+ // Entry 220 - 23F
+ 0x37a00000, 0x38000000, 0x38000117, 0x38700000,
+ 0x38900000, 0x38900131, 0x39000000, 0x3900006f,
+ 0x390000a4, 0x39500000, 0x39500099, 0x39800000,
+ 0x3980007d, 0x39800106, 0x39d00000, 0x39d05000,
+ 0x39d050e8, 0x39d33000, 0x39d33099, 0x3a100000,
+ 0x3b300000, 0x3b3000e9, 0x3bd00000, 0x3bd00001,
+ 0x3be00000, 0x3be00024, 0x3c000000, 0x3c00002a,
+ 0x3c000041, 0x3c00004e, 0x3c00005a, 0x3c000086,
+ // Entry 240 - 25F
+ 0x3c00008b, 0x3c0000b7, 0x3c0000c6, 0x3c0000d1,
+ 0x3c0000ee, 0x3c000118, 0x3c000126, 0x3c400000,
+ 0x3c40003f, 0x3c400069, 0x3c4000e4, 0x3d400000,
+ 0x3d40004e, 0x3d900000, 0x3d90003a, 0x3dc00000,
+ 0x3dc000bc, 0x3dc00104, 0x3de00000, 0x3de0012f,
+ 0x3e200000, 0x3e200047, 0x3e2000a5, 0x3e2000ae,
+ 0x3e2000bc, 0x3e200106, 0x3e200130, 0x3e500000,
+ 0x3e500107, 0x3e600000, 0x3e60012f, 0x3eb00000,
+ // Entry 260 - 27F
+ 0x3eb00106, 0x3ec00000, 0x3ec000a4, 0x3f300000,
+ 0x3f30012f, 0x3fa00000, 0x3fa000e8, 0x3fc00000,
+ 0x3fd00000, 0x3fd00072, 0x3fd000da, 0x3fd0010c,
+ 0x3ff00000, 0x3ff000d1, 0x40100000, 0x401000c3,
+ 0x40200000, 0x4020004c, 0x40700000, 0x40800000,
+ 0x40857000, 0x408570ba, 0x408dc000, 0x408dc0ba,
+ 0x40c00000, 0x40c000b3, 0x41200000, 0x41200111,
+ 0x41600000, 0x4160010f, 0x41c00000, 0x41d00000,
+ // Entry 280 - 29F
+ 0x41e00000, 0x41f00000, 0x41f00072, 0x42200000,
+ 0x42300000, 0x42300164, 0x42900000, 0x42900062,
+ 0x4290006f, 0x429000a4, 0x42900115, 0x43100000,
+ 0x43100027, 0x431000c2, 0x4310014d, 0x43200000,
+ 0x4321f000, 0x4321f033, 0x4321f0bd, 0x4321f105,
+ 0x4321f14d, 0x43257000, 0x43257033, 0x432570bd,
+ 0x43257105, 0x4325714d, 0x43700000, 0x43a00000,
+ 0x43b00000, 0x44400000, 0x44400031, 0x44400072,
+ // Entry 2A0 - 2BF
+ 0x4440010c, 0x44500000, 0x4450004b, 0x445000a4,
+ 0x4450012f, 0x44500131, 0x44e00000, 0x45000000,
+ 0x45000099, 0x450000b3, 0x450000d0, 0x4500010d,
+ 0x46100000, 0x46100099, 0x46400000, 0x464000a4,
+ 0x46400131, 0x46700000, 0x46700124, 0x46b00000,
+ 0x46b00123, 0x46f00000, 0x46f0006d, 0x46f0006f,
+ 0x47100000, 0x47600000, 0x47600127, 0x47a00000,
+ 0x48000000, 0x48200000, 0x48200129, 0x48a00000,
+ // Entry 2C0 - 2DF
+ 0x48a0005d, 0x48a0012b, 0x48e00000, 0x49400000,
+ 0x49400106, 0x4a400000, 0x4a4000d4, 0x4a900000,
+ 0x4a9000ba, 0x4ac00000, 0x4ac00053, 0x4ae00000,
+ 0x4ae00130, 0x4b400000, 0x4b400099, 0x4b4000e8,
+ 0x4bc00000, 0x4bc05000, 0x4bc05024, 0x4bc1f000,
+ 0x4bc1f137, 0x4bc57000, 0x4bc57137, 0x4be00000,
+ 0x4be57000, 0x4be570b4, 0x4bee3000, 0x4bee30b4,
+ 0x4c000000, 0x4c300000, 0x4c30013e, 0x4c900000,
+ // Entry 2E0 - 2FF
+ 0x4c900001, 0x4cc00000, 0x4cc0012f, 0x4ce00000,
+ 0x4cf00000, 0x4cf0004e, 0x4e500000, 0x4e500114,
+ 0x4f200000, 0x4fb00000, 0x4fb00131, 0x50900000,
+ 0x50900052, 0x51200000, 0x51200001, 0x51800000,
+ 0x5180003b, 0x518000d6, 0x51f00000, 0x51f38000,
+ 0x51f38053, 0x51f39000, 0x51f3908d, 0x52800000,
+ 0x528000ba, 0x52900000, 0x52938000, 0x52938053,
+ 0x5293808d, 0x529380c6, 0x5293810d, 0x52939000,
+ // Entry 300 - 31F
+ 0x5293908d, 0x529390c6, 0x5293912e, 0x52f00000,
+ 0x52f00161,
+} // Size: 3116 bytes
+
+const specialTagsStr string = "ca-ES-valencia en-US-u-va-posix"
+
+var regionToGroups = []uint8{ // 357 elements
+ // Entry 0 - 3F
+ 0x00, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x04,
+ 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x00,
+ 0x00, 0x04, 0x00, 0x00, 0x04, 0x01, 0x00, 0x00,
+ 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x00, 0x04,
+ // Entry 40 - 7F
+ 0x04, 0x04, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x04, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x04, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00,
+ 0x00, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x00, 0x08,
+ 0x00, 0x04, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00,
+ // Entry 80 - BF
+ 0x00, 0x00, 0x04, 0x00, 0x00, 0x04, 0x00, 0x00,
+ 0x00, 0x04, 0x01, 0x00, 0x04, 0x02, 0x00, 0x04,
+ 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00,
+ 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x08, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00,
+ // Entry C0 - FF
+ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x01,
+ 0x04, 0x08, 0x04, 0x00, 0x00, 0x00, 0x00, 0x04,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x04, 0x00, 0x05, 0x00, 0x00, 0x00,
+ 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ // Entry 100 - 13F
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00,
+ 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x00, 0x04,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x01, 0x00, 0x05, 0x04, 0x00,
+ 0x00, 0x04, 0x00, 0x04, 0x04, 0x05, 0x00, 0x00,
+ // Entry 140 - 17F
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00,
+} // Size: 381 bytes
+
+var paradigmLocales = [][3]uint16{ // 3 elements
+ 0: [3]uint16{0x139, 0x0, 0x7b},
+ 1: [3]uint16{0x13e, 0x0, 0x1f},
+ 2: [3]uint16{0x3c0, 0x41, 0xee},
+} // Size: 42 bytes
+
+type mutualIntelligibility struct {
+ want uint16
+ have uint16
+ distance uint8
+ oneway bool
+}
+type scriptIntelligibility struct {
+ wantLang uint16
+ haveLang uint16
+ wantScript uint8
+ haveScript uint8
+ distance uint8
+}
+type regionIntelligibility struct {
+ lang uint16
+ script uint8
+ group uint8
+ distance uint8
+}
+
+// matchLang holds pairs of langIDs of base languages that are typically
+// mutually intelligible. Each pair is associated with a confidence and
+// whether the intelligibility goes one or both ways.
+var matchLang = []mutualIntelligibility{ // 113 elements
+ 0: {want: 0x1d1, have: 0xb7, distance: 0x4, oneway: false},
+ 1: {want: 0x407, have: 0xb7, distance: 0x4, oneway: false},
+ 2: {want: 0x407, have: 0x1d1, distance: 0x4, oneway: false},
+ 3: {want: 0x407, have: 0x432, distance: 0x4, oneway: false},
+ 4: {want: 0x43a, have: 0x1, distance: 0x4, oneway: false},
+ 5: {want: 0x1a3, have: 0x10d, distance: 0x4, oneway: true},
+ 6: {want: 0x295, have: 0x10d, distance: 0x4, oneway: true},
+ 7: {want: 0x101, have: 0x36f, distance: 0x8, oneway: false},
+ 8: {want: 0x101, have: 0x347, distance: 0x8, oneway: false},
+ 9: {want: 0x5, have: 0x3e2, distance: 0xa, oneway: true},
+ 10: {want: 0xd, have: 0x139, distance: 0xa, oneway: true},
+ 11: {want: 0x16, have: 0x367, distance: 0xa, oneway: true},
+ 12: {want: 0x21, have: 0x139, distance: 0xa, oneway: true},
+ 13: {want: 0x56, have: 0x13e, distance: 0xa, oneway: true},
+ 14: {want: 0x58, have: 0x3e2, distance: 0xa, oneway: true},
+ 15: {want: 0x71, have: 0x3e2, distance: 0xa, oneway: true},
+ 16: {want: 0x75, have: 0x139, distance: 0xa, oneway: true},
+ 17: {want: 0x82, have: 0x1be, distance: 0xa, oneway: true},
+ 18: {want: 0xa5, have: 0x139, distance: 0xa, oneway: true},
+ 19: {want: 0xb2, have: 0x15e, distance: 0xa, oneway: true},
+ 20: {want: 0xdd, have: 0x153, distance: 0xa, oneway: true},
+ 21: {want: 0xe5, have: 0x139, distance: 0xa, oneway: true},
+ 22: {want: 0xe9, have: 0x3a, distance: 0xa, oneway: true},
+ 23: {want: 0xf0, have: 0x15e, distance: 0xa, oneway: true},
+ 24: {want: 0xf9, have: 0x15e, distance: 0xa, oneway: true},
+ 25: {want: 0x100, have: 0x139, distance: 0xa, oneway: true},
+ 26: {want: 0x130, have: 0x139, distance: 0xa, oneway: true},
+ 27: {want: 0x13c, have: 0x139, distance: 0xa, oneway: true},
+ 28: {want: 0x140, have: 0x151, distance: 0xa, oneway: true},
+ 29: {want: 0x145, have: 0x13e, distance: 0xa, oneway: true},
+ 30: {want: 0x158, have: 0x101, distance: 0xa, oneway: true},
+ 31: {want: 0x16d, have: 0x367, distance: 0xa, oneway: true},
+ 32: {want: 0x16e, have: 0x139, distance: 0xa, oneway: true},
+ 33: {want: 0x16f, have: 0x139, distance: 0xa, oneway: true},
+ 34: {want: 0x17e, have: 0x139, distance: 0xa, oneway: true},
+ 35: {want: 0x190, have: 0x13e, distance: 0xa, oneway: true},
+ 36: {want: 0x194, have: 0x13e, distance: 0xa, oneway: true},
+ 37: {want: 0x1a4, have: 0x1be, distance: 0xa, oneway: true},
+ 38: {want: 0x1b4, have: 0x139, distance: 0xa, oneway: true},
+ 39: {want: 0x1b8, have: 0x139, distance: 0xa, oneway: true},
+ 40: {want: 0x1d4, have: 0x15e, distance: 0xa, oneway: true},
+ 41: {want: 0x1d7, have: 0x3e2, distance: 0xa, oneway: true},
+ 42: {want: 0x1d9, have: 0x139, distance: 0xa, oneway: true},
+ 43: {want: 0x1e7, have: 0x139, distance: 0xa, oneway: true},
+ 44: {want: 0x1f8, have: 0x139, distance: 0xa, oneway: true},
+ 45: {want: 0x20e, have: 0x1e1, distance: 0xa, oneway: true},
+ 46: {want: 0x210, have: 0x139, distance: 0xa, oneway: true},
+ 47: {want: 0x22d, have: 0x15e, distance: 0xa, oneway: true},
+ 48: {want: 0x242, have: 0x3e2, distance: 0xa, oneway: true},
+ 49: {want: 0x24a, have: 0x139, distance: 0xa, oneway: true},
+ 50: {want: 0x251, have: 0x139, distance: 0xa, oneway: true},
+ 51: {want: 0x265, have: 0x139, distance: 0xa, oneway: true},
+ 52: {want: 0x274, have: 0x48a, distance: 0xa, oneway: true},
+ 53: {want: 0x28a, have: 0x3e2, distance: 0xa, oneway: true},
+ 54: {want: 0x28e, have: 0x1f9, distance: 0xa, oneway: true},
+ 55: {want: 0x2a3, have: 0x139, distance: 0xa, oneway: true},
+ 56: {want: 0x2b5, have: 0x15e, distance: 0xa, oneway: true},
+ 57: {want: 0x2b8, have: 0x139, distance: 0xa, oneway: true},
+ 58: {want: 0x2be, have: 0x139, distance: 0xa, oneway: true},
+ 59: {want: 0x2c3, have: 0x15e, distance: 0xa, oneway: true},
+ 60: {want: 0x2ed, have: 0x139, distance: 0xa, oneway: true},
+ 61: {want: 0x2f1, have: 0x15e, distance: 0xa, oneway: true},
+ 62: {want: 0x2fa, have: 0x139, distance: 0xa, oneway: true},
+ 63: {want: 0x2ff, have: 0x7e, distance: 0xa, oneway: true},
+ 64: {want: 0x304, have: 0x139, distance: 0xa, oneway: true},
+ 65: {want: 0x30b, have: 0x3e2, distance: 0xa, oneway: true},
+ 66: {want: 0x31b, have: 0x1be, distance: 0xa, oneway: true},
+ 67: {want: 0x31f, have: 0x1e1, distance: 0xa, oneway: true},
+ 68: {want: 0x320, have: 0x139, distance: 0xa, oneway: true},
+ 69: {want: 0x331, have: 0x139, distance: 0xa, oneway: true},
+ 70: {want: 0x351, have: 0x139, distance: 0xa, oneway: true},
+ 71: {want: 0x36a, have: 0x347, distance: 0xa, oneway: false},
+ 72: {want: 0x36a, have: 0x36f, distance: 0xa, oneway: true},
+ 73: {want: 0x37a, have: 0x139, distance: 0xa, oneway: true},
+ 74: {want: 0x387, have: 0x139, distance: 0xa, oneway: true},
+ 75: {want: 0x389, have: 0x139, distance: 0xa, oneway: true},
+ 76: {want: 0x38b, have: 0x15e, distance: 0xa, oneway: true},
+ 77: {want: 0x390, have: 0x139, distance: 0xa, oneway: true},
+ 78: {want: 0x395, have: 0x139, distance: 0xa, oneway: true},
+ 79: {want: 0x39d, have: 0x139, distance: 0xa, oneway: true},
+ 80: {want: 0x3a5, have: 0x139, distance: 0xa, oneway: true},
+ 81: {want: 0x3be, have: 0x139, distance: 0xa, oneway: true},
+ 82: {want: 0x3c4, have: 0x13e, distance: 0xa, oneway: true},
+ 83: {want: 0x3d4, have: 0x10d, distance: 0xa, oneway: true},
+ 84: {want: 0x3d9, have: 0x139, distance: 0xa, oneway: true},
+ 85: {want: 0x3e5, have: 0x15e, distance: 0xa, oneway: true},
+ 86: {want: 0x3e9, have: 0x1be, distance: 0xa, oneway: true},
+ 87: {want: 0x3fa, have: 0x139, distance: 0xa, oneway: true},
+ 88: {want: 0x40c, have: 0x139, distance: 0xa, oneway: true},
+ 89: {want: 0x423, have: 0x139, distance: 0xa, oneway: true},
+ 90: {want: 0x429, have: 0x139, distance: 0xa, oneway: true},
+ 91: {want: 0x431, have: 0x139, distance: 0xa, oneway: true},
+ 92: {want: 0x43b, have: 0x139, distance: 0xa, oneway: true},
+ 93: {want: 0x43e, have: 0x1e1, distance: 0xa, oneway: true},
+ 94: {want: 0x445, have: 0x139, distance: 0xa, oneway: true},
+ 95: {want: 0x450, have: 0x139, distance: 0xa, oneway: true},
+ 96: {want: 0x461, have: 0x139, distance: 0xa, oneway: true},
+ 97: {want: 0x467, have: 0x3e2, distance: 0xa, oneway: true},
+ 98: {want: 0x46f, have: 0x139, distance: 0xa, oneway: true},
+ 99: {want: 0x476, have: 0x3e2, distance: 0xa, oneway: true},
+ 100: {want: 0x3883, have: 0x139, distance: 0xa, oneway: true},
+ 101: {want: 0x480, have: 0x139, distance: 0xa, oneway: true},
+ 102: {want: 0x482, have: 0x139, distance: 0xa, oneway: true},
+ 103: {want: 0x494, have: 0x3e2, distance: 0xa, oneway: true},
+ 104: {want: 0x49d, have: 0x139, distance: 0xa, oneway: true},
+ 105: {want: 0x4ac, have: 0x529, distance: 0xa, oneway: true},
+ 106: {want: 0x4b4, have: 0x139, distance: 0xa, oneway: true},
+ 107: {want: 0x4bc, have: 0x3e2, distance: 0xa, oneway: true},
+ 108: {want: 0x4e5, have: 0x15e, distance: 0xa, oneway: true},
+ 109: {want: 0x4f2, have: 0x139, distance: 0xa, oneway: true},
+ 110: {want: 0x512, have: 0x139, distance: 0xa, oneway: true},
+ 111: {want: 0x518, have: 0x139, distance: 0xa, oneway: true},
+ 112: {want: 0x52f, have: 0x139, distance: 0xa, oneway: true},
+} // Size: 702 bytes
+
+// matchScript holds pairs of scriptIDs where readers of one script
+// can typically also read the other. Each is associated with a confidence.
+var matchScript = []scriptIntelligibility{ // 26 elements
+ 0: {wantLang: 0x432, haveLang: 0x432, wantScript: 0x57, haveScript: 0x1f, distance: 0x5},
+ 1: {wantLang: 0x432, haveLang: 0x432, wantScript: 0x1f, haveScript: 0x57, distance: 0x5},
+ 2: {wantLang: 0x58, haveLang: 0x3e2, wantScript: 0x57, haveScript: 0x1f, distance: 0xa},
+ 3: {wantLang: 0xa5, haveLang: 0x139, wantScript: 0xe, haveScript: 0x57, distance: 0xa},
+ 4: {wantLang: 0x1d7, haveLang: 0x3e2, wantScript: 0x8, haveScript: 0x1f, distance: 0xa},
+ 5: {wantLang: 0x210, haveLang: 0x139, wantScript: 0x2b, haveScript: 0x57, distance: 0xa},
+ 6: {wantLang: 0x24a, haveLang: 0x139, wantScript: 0x4b, haveScript: 0x57, distance: 0xa},
+ 7: {wantLang: 0x251, haveLang: 0x139, wantScript: 0x4f, haveScript: 0x57, distance: 0xa},
+ 8: {wantLang: 0x2b8, haveLang: 0x139, wantScript: 0x54, haveScript: 0x57, distance: 0xa},
+ 9: {wantLang: 0x304, haveLang: 0x139, wantScript: 0x6b, haveScript: 0x57, distance: 0xa},
+ 10: {wantLang: 0x331, haveLang: 0x139, wantScript: 0x72, haveScript: 0x57, distance: 0xa},
+ 11: {wantLang: 0x351, haveLang: 0x139, wantScript: 0x21, haveScript: 0x57, distance: 0xa},
+ 12: {wantLang: 0x395, haveLang: 0x139, wantScript: 0x7d, haveScript: 0x57, distance: 0xa},
+ 13: {wantLang: 0x39d, haveLang: 0x139, wantScript: 0x33, haveScript: 0x57, distance: 0xa},
+ 14: {wantLang: 0x3be, haveLang: 0x139, wantScript: 0x5, haveScript: 0x57, distance: 0xa},
+ 15: {wantLang: 0x3fa, haveLang: 0x139, wantScript: 0x5, haveScript: 0x57, distance: 0xa},
+ 16: {wantLang: 0x40c, haveLang: 0x139, wantScript: 0xca, haveScript: 0x57, distance: 0xa},
+ 17: {wantLang: 0x450, haveLang: 0x139, wantScript: 0xd7, haveScript: 0x57, distance: 0xa},
+ 18: {wantLang: 0x461, haveLang: 0x139, wantScript: 0xda, haveScript: 0x57, distance: 0xa},
+ 19: {wantLang: 0x46f, haveLang: 0x139, wantScript: 0x29, haveScript: 0x57, distance: 0xa},
+ 20: {wantLang: 0x476, haveLang: 0x3e2, wantScript: 0x57, haveScript: 0x1f, distance: 0xa},
+ 21: {wantLang: 0x4b4, haveLang: 0x139, wantScript: 0x5, haveScript: 0x57, distance: 0xa},
+ 22: {wantLang: 0x4bc, haveLang: 0x3e2, wantScript: 0x57, haveScript: 0x1f, distance: 0xa},
+ 23: {wantLang: 0x512, haveLang: 0x139, wantScript: 0x3b, haveScript: 0x57, distance: 0xa},
+ 24: {wantLang: 0x529, haveLang: 0x529, wantScript: 0x38, haveScript: 0x39, distance: 0xf},
+ 25: {wantLang: 0x529, haveLang: 0x529, wantScript: 0x39, haveScript: 0x38, distance: 0x13},
+} // Size: 232 bytes
+
+var matchRegion = []regionIntelligibility{ // 15 elements
+ 0: {lang: 0x3a, script: 0x0, group: 0x4, distance: 0x4},
+ 1: {lang: 0x3a, script: 0x0, group: 0x84, distance: 0x4},
+ 2: {lang: 0x139, script: 0x0, group: 0x1, distance: 0x4},
+ 3: {lang: 0x139, script: 0x0, group: 0x81, distance: 0x4},
+ 4: {lang: 0x13e, script: 0x0, group: 0x3, distance: 0x4},
+ 5: {lang: 0x13e, script: 0x0, group: 0x83, distance: 0x4},
+ 6: {lang: 0x3c0, script: 0x0, group: 0x3, distance: 0x4},
+ 7: {lang: 0x3c0, script: 0x0, group: 0x83, distance: 0x4},
+ 8: {lang: 0x529, script: 0x39, group: 0x2, distance: 0x4},
+ 9: {lang: 0x529, script: 0x39, group: 0x82, distance: 0x4},
+ 10: {lang: 0x3a, script: 0x0, group: 0x80, distance: 0x5},
+ 11: {lang: 0x139, script: 0x0, group: 0x80, distance: 0x5},
+ 12: {lang: 0x13e, script: 0x0, group: 0x80, distance: 0x5},
+ 13: {lang: 0x3c0, script: 0x0, group: 0x80, distance: 0x5},
+ 14: {lang: 0x529, script: 0x39, group: 0x80, distance: 0x5},
+} // Size: 114 bytes
+
+// Total table size 4618 bytes (4KiB); checksum: D161A896
diff --git a/internal/language/compact/tags.go b/internal/language/compact/tags.go
new file mode 100644
index 0000000..acc482a
--- /dev/null
+++ b/internal/language/compact/tags.go
@@ -0,0 +1,143 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+// TODO: Various sets of commonly use tags and regions.
+
+// MustParse is like Parse, but panics if the given BCP 47 tag cannot be parsed.
+// It simplifies safe initialization of Tag values.
+func MustParse(s string) Tag {
+ t, err := Parse(s)
+ if err != nil {
+ panic(err)
+ }
+ return t
+}
+
+// MustParse is like Parse, but panics if the given BCP 47 tag cannot be parsed.
+// It simplifies safe initialization of Tag values.
+func (c CanonType) MustParse(s string) Tag {
+ t, err := c.Parse(s)
+ if err != nil {
+ panic(err)
+ }
+ return t
+}
+
+// MustParseBase is like ParseBase, but panics if the given base cannot be parsed.
+// It simplifies safe initialization of Base values.
+func MustParseBase(s string) Base {
+ b, err := ParseBase(s)
+ if err != nil {
+ panic(err)
+ }
+ return b
+}
+
+// MustParseScript is like ParseScript, but panics if the given script cannot be
+// parsed. It simplifies safe initialization of Script values.
+func MustParseScript(s string) Script {
+ scr, err := ParseScript(s)
+ if err != nil {
+ panic(err)
+ }
+ return scr
+}
+
+// MustParseRegion is like ParseRegion, but panics if the given region cannot be
+// parsed. It simplifies safe initialization of Region values.
+func MustParseRegion(s string) Region {
+ r, err := ParseRegion(s)
+ if err != nil {
+ panic(err)
+ }
+ return r
+}
+
+var (
+ und = Tag{}
+
+ Und Tag = Tag{}
+
+ Afrikaans Tag = Tag{language: afIndex, locale: afIndex}
+ Amharic Tag = Tag{language: amIndex, locale: amIndex}
+ Arabic Tag = Tag{language: arIndex, locale: arIndex}
+ ModernStandardArabic Tag = Tag{language: ar001Index, locale: ar001Index}
+ Azerbaijani Tag = Tag{language: azIndex, locale: azIndex}
+ Bulgarian Tag = Tag{language: bgIndex, locale: bgIndex}
+ Bengali Tag = Tag{language: bnIndex, locale: bnIndex}
+ Catalan Tag = Tag{language: caIndex, locale: caIndex}
+ Czech Tag = Tag{language: csIndex, locale: csIndex}
+ Danish Tag = Tag{language: daIndex, locale: daIndex}
+ German Tag = Tag{language: deIndex, locale: deIndex}
+ Greek Tag = Tag{language: elIndex, locale: elIndex}
+ English Tag = Tag{language: enIndex, locale: enIndex}
+ AmericanEnglish Tag = Tag{language: enUSIndex, locale: enUSIndex}
+ BritishEnglish Tag = Tag{language: enGBIndex, locale: enGBIndex}
+ Spanish Tag = Tag{language: esIndex, locale: esIndex}
+ EuropeanSpanish Tag = Tag{language: esESIndex, locale: esESIndex}
+ LatinAmericanSpanish Tag = Tag{language: es419Index, locale: es419Index}
+ Estonian Tag = Tag{language: etIndex, locale: etIndex}
+ Persian Tag = Tag{language: faIndex, locale: faIndex}
+ Finnish Tag = Tag{language: fiIndex, locale: fiIndex}
+ Filipino Tag = Tag{language: filIndex, locale: filIndex}
+ French Tag = Tag{language: frIndex, locale: frIndex}
+ CanadianFrench Tag = Tag{language: frCAIndex, locale: frCAIndex}
+ Gujarati Tag = Tag{language: guIndex, locale: guIndex}
+ Hebrew Tag = Tag{language: heIndex, locale: heIndex}
+ Hindi Tag = Tag{language: hiIndex, locale: hiIndex}
+ Croatian Tag = Tag{language: hrIndex, locale: hrIndex}
+ Hungarian Tag = Tag{language: huIndex, locale: huIndex}
+ Armenian Tag = Tag{language: hyIndex, locale: hyIndex}
+ Indonesian Tag = Tag{language: idIndex, locale: idIndex}
+ Icelandic Tag = Tag{language: isIndex, locale: isIndex}
+ Italian Tag = Tag{language: itIndex, locale: itIndex}
+ Japanese Tag = Tag{language: jaIndex, locale: jaIndex}
+ Georgian Tag = Tag{language: kaIndex, locale: kaIndex}
+ Kazakh Tag = Tag{language: kkIndex, locale: kkIndex}
+ Khmer Tag = Tag{language: kmIndex, locale: kmIndex}
+ Kannada Tag = Tag{language: knIndex, locale: knIndex}
+ Korean Tag = Tag{language: koIndex, locale: koIndex}
+ Kirghiz Tag = Tag{language: kyIndex, locale: kyIndex}
+ Lao Tag = Tag{language: loIndex, locale: loIndex}
+ Lithuanian Tag = Tag{language: ltIndex, locale: ltIndex}
+ Latvian Tag = Tag{language: lvIndex, locale: lvIndex}
+ Macedonian Tag = Tag{language: mkIndex, locale: mkIndex}
+ Malayalam Tag = Tag{language: mlIndex, locale: mlIndex}
+ Mongolian Tag = Tag{language: mnIndex, locale: mnIndex}
+ Marathi Tag = Tag{language: mrIndex, locale: mrIndex}
+ Malay Tag = Tag{language: msIndex, locale: msIndex}
+ Burmese Tag = Tag{language: myIndex, locale: myIndex}
+ Nepali Tag = Tag{language: neIndex, locale: neIndex}
+ Dutch Tag = Tag{language: nlIndex, locale: nlIndex}
+ Norwegian Tag = Tag{language: noIndex, locale: noIndex}
+ Punjabi Tag = Tag{language: paIndex, locale: paIndex}
+ Polish Tag = Tag{language: plIndex, locale: plIndex}
+ Portuguese Tag = Tag{language: ptIndex, locale: ptIndex}
+ BrazilianPortuguese Tag = Tag{language: ptBRIndex, locale: ptBRIndex}
+ EuropeanPortuguese Tag = Tag{language: ptPTIndex, locale: ptPTIndex}
+ Romanian Tag = Tag{language: roIndex, locale: roIndex}
+ Russian Tag = Tag{language: ruIndex, locale: ruIndex}
+ Sinhala Tag = Tag{language: siIndex, locale: siIndex}
+ Slovak Tag = Tag{language: skIndex, locale: skIndex}
+ Slovenian Tag = Tag{language: slIndex, locale: slIndex}
+ Albanian Tag = Tag{language: sqIndex, locale: sqIndex}
+ Serbian Tag = Tag{language: srIndex, locale: srIndex}
+ SerbianLatin Tag = Tag{language: srLatnIndex, locale: srLatnIndex}
+ Swedish Tag = Tag{language: svIndex, locale: svIndex}
+ Swahili Tag = Tag{language: swIndex, locale: swIndex}
+ Tamil Tag = Tag{language: taIndex, locale: taIndex}
+ Telugu Tag = Tag{language: teIndex, locale: teIndex}
+ Thai Tag = Tag{language: thIndex, locale: thIndex}
+ Turkish Tag = Tag{language: trIndex, locale: trIndex}
+ Ukrainian Tag = Tag{language: ukIndex, locale: ukIndex}
+ Urdu Tag = Tag{language: urIndex, locale: urIndex}
+ Uzbek Tag = Tag{language: uzIndex, locale: uzIndex}
+ Vietnamese Tag = Tag{language: viIndex, locale: viIndex}
+ Chinese Tag = Tag{language: zhIndex, locale: zhIndex}
+ SimplifiedChinese Tag = Tag{language: zhHansIndex, locale: zhHansIndex}
+ TraditionalChinese Tag = Tag{language: zhHantIndex, locale: zhHantIndex}
+ Zulu Tag = Tag{language: zuIndex, locale: zuIndex}
+)
