| // Copyright 2013 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| package language |
| |
| import ( |
| "bytes" |
| "flag" |
| "fmt" |
| "os" |
| "path" |
| "path/filepath" |
| "strings" |
| "testing" |
| "unicode/utf8" |
| |
| "golang.org/x/text/internal/testtext" |
| "golang.org/x/text/internal/ucd" |
| ) |
| |
| var verbose = flag.Bool("verbose", false, "set to true to print the internal tables of matchers") |
| |
| func TestCompliance(t *testing.T) { |
| filepath.Walk("testdata", func(file string, info os.FileInfo, err error) error { |
| if info.IsDir() { |
| return nil |
| } |
| r, err := os.Open(file) |
| if err != nil { |
| t.Fatal(err) |
| } |
| ucd.Parse(r, func(p *ucd.Parser) { |
| name := strings.ReplaceAll(path.Join(p.String(0), p.String(1)), " ", "") |
| if skip[name] { |
| return |
| } |
| t.Run(info.Name()+"/"+short(name), func(t *testing.T) { |
| supported := makeTagList(p.String(0)) |
| desired := makeTagList(p.String(1)) |
| gotCombined, index, conf := NewMatcher(supported).Match(desired...) |
| |
| gotMatch := supported[index] |
| wantMatch := Raw.Make(p.String(2)) // wantMatch may be null |
| if gotMatch != wantMatch { |
| t.Fatalf("match: got %q; want %q (%v)", gotMatch, wantMatch, conf) |
| } |
| if tag := strings.TrimSpace(p.String(3)); tag != "" { |
| wantCombined := Raw.MustParse(tag) |
| if err == nil && gotCombined != wantCombined { |
| t.Errorf("combined: got %q; want %q (%v)", gotCombined, wantCombined, conf) |
| } |
| } |
| }) |
| }) |
| return nil |
| }) |
| } |
| |
| func short(s string) string { |
| if len(s) <= 50 { |
| return s |
| } |
| var i int |
| for i = 1; i < utf8.UTFMax && !utf8.RuneStart(s[50-i]); i++ { |
| } |
| return s[:50-i] + "…" |
| } |
| |
| var skip = map[string]bool{ |
| // TODO: bugs |
| // Honor the wildcard match. This may only be useful to select non-exact |
| // stuff. |
| "mul,af/nl": true, // match: got "af"; want "mul" |
| |
| // TODO: include other extensions. |
| // combined: got "en-GB-u-ca-buddhist-nu-arab"; want "en-GB-fonipa-t-m0-iso-i0-pinyin-u-ca-buddhist-nu-arab" |
| "und,en-GB-u-sd-gbsct/en-fonipa-u-nu-Arab-ca-buddhist-t-m0-iso-i0-pinyin": true, |
| |
| // Inconsistencies with Mark Davis' implementation where it is not clear |
| // which is better. |
| |
| // Inconsistencies in combined. I think the Go approach is more appropriate. |
| // We could use -u-rg- as alternative. |
| "und,fr/fr-BE-fonipa": true, // combined: got "fr"; want "fr-BE-fonipa" |
| "und,fr-CA/fr-BE-fonipa": true, // combined: got "fr-CA"; want "fr-BE-fonipa" |
| "und,fr-fonupa/fr-BE-fonipa": true, // combined: got "fr-fonupa"; want "fr-BE-fonipa" |
| "und,no/nn-BE-fonipa": true, // combined: got "no"; want "no-BE-fonipa" |
| "50,und,fr-CA-fonupa/fr-BE-fonipa": true, // combined: got "fr-CA-fonupa"; want "fr-BE-fonipa" |
| |
| // The initial number is a threshold. As we don't use scoring, we will not |
| // implement this. |
| "50,und,fr-Cyrl-CA-fonupa/fr-BE-fonipa": true, |
| // match: got "und"; want "fr-Cyrl-CA-fonupa" |
| // combined: got "und"; want "fr-Cyrl-BE-fonipa" |
| |
| // Other interesting cases to test: |
| // - Should same language or same script have the preference if there is |
| // usually no understanding of the other script? |
| // - More specific region in desired may replace enclosing supported. |
| } |
| |
| func makeTagList(s string) (tags []Tag) { |
| for _, s := range strings.Split(s, ",") { |
| tags = append(tags, mk(strings.TrimSpace(s))) |
| } |
| return tags |
| } |
| |
| func TestMatchStrings(t *testing.T) { |
| testCases := []struct { |
| supported string |
| desired string // strings separated by | |
| tag string |
| index int |
| }{{ |
| supported: "en", |
| desired: "", |
| tag: "en", |
| index: 0, |
| }, { |
| supported: "en", |
| desired: "nl", |
| tag: "en", |
| index: 0, |
| }, { |
| supported: "en,nl", |
| desired: "nl", |
| tag: "nl", |
| index: 1, |
| }, { |
| supported: "en,nl", |
| desired: "nl|en", |
| tag: "nl", |
| index: 1, |
| }, { |
| supported: "en-GB,nl", |
| desired: "en ; q=0.1,nl", |
| tag: "nl", |
| index: 1, |
| }, { |
| supported: "en-GB,nl", |
| desired: "en;q=0.005 | dk; q=0.1,nl ", |
| tag: "en-GB", |
| index: 0, |
| }, { |
| // do not match faulty tags with und |
| supported: "en,und", |
| desired: "|en", |
| tag: "en", |
| index: 0, |
| }} |
| for _, tc := range testCases { |
| t.Run(path.Join(tc.supported, tc.desired), func(t *testing.T) { |
| m := NewMatcher(makeTagList(tc.supported)) |
| tag, index := MatchStrings(m, strings.Split(tc.desired, "|")...) |
| if tag.String() != tc.tag || index != tc.index { |
| t.Errorf("got %v, %d; want %v, %d", tag, index, tc.tag, tc.index) |
| } |
| }) |
| } |
| } |
| |
| func TestRegionGroups(t *testing.T) { |
| testCases := []struct { |
| a, b string |
| distance uint8 |
| }{ |
| {"zh-TW", "zh-HK", 5}, |
| {"zh-MO", "zh-HK", 4}, |
| {"es-ES", "es-AR", 5}, |
| {"es-ES", "es", 4}, |
| {"es-419", "es-MX", 4}, |
| {"es-AR", "es-MX", 4}, |
| {"es-ES", "es-MX", 5}, |
| {"es-PT", "es-MX", 5}, |
| } |
| for _, tc := range testCases { |
| a := MustParse(tc.a) |
| aScript, _ := a.Script() |
| b := MustParse(tc.b) |
| bScript, _ := b.Script() |
| |
| if aScript != bScript { |
| t.Errorf("scripts differ: %q vs %q", aScript, bScript) |
| continue |
| } |
| d, _ := regionGroupDist(a.region(), b.region(), aScript.scriptID, a.lang()) |
| if d != tc.distance { |
| t.Errorf("got %q; want %q", d, tc.distance) |
| } |
| } |
| } |
| |
| func TestIsParadigmLocale(t *testing.T) { |
| testCases := map[string]bool{ |
| "en-US": true, |
| "en-GB": true, |
| "en-VI": false, |
| "es-GB": false, |
| "es-ES": true, |
| "es-419": true, |
| } |
| for str, want := range testCases { |
| tt := Make(str) |
| tag := tt.tag() |
| got := isParadigmLocale(tag.LangID, tag.RegionID) |
| if got != want { |
| t.Errorf("isPL(%q) = %v; want %v", str, got, want) |
| } |
| } |
| } |
| |
| // Implementation of String methods for various types for debugging purposes. |
| |
| func (m *matcher) String() string { |
| w := &bytes.Buffer{} |
| fmt.Fprintln(w, "Default:", m.default_) |
| for tag, h := range m.index { |
| fmt.Fprintf(w, " %s: %v\n", tag, h) |
| } |
| return w.String() |
| } |
| |
| func (h *matchHeader) String() string { |
| w := &bytes.Buffer{} |
| fmt.Fprint(w, "haveTag: ") |
| for _, h := range h.haveTags { |
| fmt.Fprintf(w, "%v, ", h) |
| } |
| return w.String() |
| } |
| |
| func (t haveTag) String() string { |
| return fmt.Sprintf("%v:%d:%v:%v-%v|%v", t.tag, t.index, t.conf, t.maxRegion, t.maxScript, t.altScript) |
| } |
| |
| func TestIssue43834(t *testing.T) { |
| matcher := NewMatcher([]Tag{English}) |
| |
| // ZZ is the largest region code and should not cause overflow. |
| desired, _, err := ParseAcceptLanguage("en-ZZ") |
| if err != nil { |
| t.Error(err) |
| } |
| _, i, _ := matcher.Match(desired...) |
| if i != 0 { |
| t.Errorf("got %v; want 0", i) |
| } |
| } |
| |
| func TestBestMatchAlloc(t *testing.T) { |
| m := NewMatcher(makeTagList("en sr nl")) |
| // Go allocates when creating a list of tags from a single tag! |
| list := []Tag{English} |
| avg := testtext.AllocsPerRun(100, func() { |
| m.Match(list...) |
| }) |
| if avg > 0 { |
| t.Errorf("got %f; want 0", avg) |
| } |
| } |
| |
| var benchHave = []Tag{ |
| mk("en"), |
| mk("en-GB"), |
| mk("za"), |
| mk("zh-Hant"), |
| mk("zh-Hans-CN"), |
| mk("zh"), |
| mk("zh-HK"), |
| mk("ar-MK"), |
| mk("en-CA"), |
| mk("fr-CA"), |
| mk("fr-US"), |
| mk("fr-CH"), |
| mk("fr"), |
| mk("lt"), |
| mk("lv"), |
| mk("iw"), |
| mk("iw-NL"), |
| mk("he"), |
| mk("he-IT"), |
| mk("tlh"), |
| mk("ja"), |
| mk("ja-Jpan"), |
| mk("ja-Jpan-JP"), |
| mk("de"), |
| mk("de-CH"), |
| mk("de-AT"), |
| mk("de-DE"), |
| mk("sr"), |
| mk("sr-Latn"), |
| mk("sr-Cyrl"), |
| mk("sr-ME"), |
| } |
| |
| var benchWant = [][]Tag{ |
| []Tag{ |
| mk("en"), |
| }, |
| []Tag{ |
| mk("en-AU"), |
| mk("de-HK"), |
| mk("nl"), |
| mk("fy"), |
| mk("lv"), |
| }, |
| []Tag{ |
| mk("en-AU"), |
| mk("de-HK"), |
| mk("nl"), |
| mk("fy"), |
| }, |
| []Tag{ |
| mk("ja-Hant"), |
| mk("da-HK"), |
| mk("nl"), |
| mk("zh-TW"), |
| }, |
| []Tag{ |
| mk("ja-Hant"), |
| mk("da-HK"), |
| mk("nl"), |
| mk("hr"), |
| }, |
| } |
| |
| func BenchmarkMatch(b *testing.B) { |
| m := newMatcher(benchHave, nil) |
| for i := 0; i < b.N; i++ { |
| for _, want := range benchWant { |
| m.getBest(want...) |
| } |
| } |
| } |
| |
| func BenchmarkMatchExact(b *testing.B) { |
| want := mk("en") |
| m := newMatcher(benchHave, nil) |
| for i := 0; i < b.N; i++ { |
| m.getBest(want) |
| } |
| } |
| |
| func BenchmarkMatchAltLanguagePresent(b *testing.B) { |
| want := mk("hr") |
| m := newMatcher(benchHave, nil) |
| for i := 0; i < b.N; i++ { |
| m.getBest(want) |
| } |
| } |
| |
| func BenchmarkMatchAltLanguageNotPresent(b *testing.B) { |
| want := mk("nn") |
| m := newMatcher(benchHave, nil) |
| for i := 0; i < b.N; i++ { |
| m.getBest(want) |
| } |
| } |
| |
| func BenchmarkMatchAltScriptPresent(b *testing.B) { |
| want := mk("zh-Hant-CN") |
| m := newMatcher(benchHave, nil) |
| for i := 0; i < b.N; i++ { |
| m.getBest(want) |
| } |
| } |
| |
| func BenchmarkMatchAltScriptNotPresent(b *testing.B) { |
| want := mk("fr-Cyrl") |
| m := newMatcher(benchHave, nil) |
| for i := 0; i < b.N; i++ { |
| m.getBest(want) |
| } |
| } |
| |
| func BenchmarkMatchLimitedExact(b *testing.B) { |
| want := []Tag{mk("he-NL"), mk("iw-NL")} |
| m := newMatcher(benchHave, nil) |
| for i := 0; i < b.N; i++ { |
| m.getBest(want...) |
| } |
| } |