language: distinguish regional variant in matching
Change-Id: I1d997d625726e4b8152a7d039fc761dfcad45b7a
Reviewed-on: https://go-review.googlesource.com/95832
Run-TryBot: Marcel van Lohuizen <mpvl@golang.org>
Reviewed-by: Nigel Tao <nigeltao@golang.org>
diff --git a/language/httpexample_test.go b/language/httpexample_test.go
index 40d0663..03c0ab9 100644
--- a/language/httpexample_test.go
+++ b/language/httpexample_test.go
@@ -24,7 +24,7 @@
t, q, err := language.ParseAcceptLanguage(r.Header.Get("Accept-Language"))
// We ignore the error: the default language will be selected for t == nil.
tag, _, _ := matcher.Match(t...)
- fmt.Printf("%5v (t: %6v; q: %3v; err: %v)\n", tag, t, q, err)
+ fmt.Printf("%17v (t: %6v; q: %3v; err: %v)\n", tag, t, q, err)
}
func ExampleParseAcceptLanguage() {
@@ -41,8 +41,8 @@
}
// Output:
- // en-GB (t: [ en en-US nn]; q: [ 1 0.8 0.3]; err: <nil>)
- // en-GB (t: [ gsw en-US en]; q: [ 1 0.8 0.7]; err: <nil>)
- // de (t: [ gsw nl da]; q: [ 1 1 1]; err: <nil>)
- // en-GB (t: []; q: []; err: language: tag is not well-formed)
+ // en-GB (t: [ en en-US nn]; q: [ 1 0.8 0.3]; err: <nil>)
+ // en-GB-u-rg-uszzzz (t: [ gsw en-US en]; q: [ 1 0.8 0.7]; err: <nil>)
+ // de (t: [ gsw nl da]; q: [ 1 1 1]; err: <nil>)
+ // en-GB (t: []; q: []; err: language: tag is not well-formed)
}
diff --git a/language/language.go b/language/language.go
index 9ddff22..b254794 100644
--- a/language/language.go
+++ b/language/language.go
@@ -30,10 +30,11 @@
}
func makeTag(t language.Tag) (tag Tag) {
- if region := t.TypeForKey("rg"); len(region) > 2 {
+ if region := t.TypeForKey("rg"); len(region) == 6 && region[2:] == "zzzz" {
if r, err := language.ParseRegion(region[:2]); err == nil {
tFull := t
t, _ = t.SetTypeForKey("rg", "")
+ // TODO: should we not consider "va" for the language tag?
var exact1, exact2 bool
tag.language, exact1 = compactIndex(t)
t.RegionID = r
@@ -60,7 +61,7 @@
tag := t.language.tag()
if t.language != t.locale {
loc := t.locale.tag()
- tag.SetTypeForKey("rg", strings.ToLower(loc.RegionID.String())+"zzzz")
+ tag, _ = tag.SetTypeForKey("rg", strings.ToLower(loc.RegionID.String())+"zzzz")
}
return tag
}
@@ -483,12 +484,56 @@
}
// CompactIndex returns an index, where 0 <= index < NumCompactTags, for tags
-// for which data exists in the text repository. The index will change over time
+// for which data exists in the text repository.The index will change over time
// and should not be stored in persistent storage. If t does not match a compact
// index, exact will be false and the compact index will be returned for the
// first match after repeatedly taking the Parent of t.
func CompactIndex(t Tag) (index int, exact bool) {
- return int(t.locale), t.language == t.locale && t.full == nil
+ return int(t.language), t.full == nil
+}
+
+// TODO: make these functions and methods public once we settle on the API and
+//
+
+// regionalCompactIndex returns the CompactIndex for the regional variant of
+// this tag. This index is used to indicate region-specific overrides, such as
+// default currency, default calendar and week data, default time cycle, and
+// default measurement system and unit preferences.
+//
+// For instance, the tag en-GB-u-rg-uszzzz specifies British English with US
+// settings for currency, number formatting, etc. The CompactIndex for this tag
+// will be that for en-GB, while the regionalCompactIndex will be the one
+// corresponding to en-US.
+func regionalCompactIndex(t Tag) (index int, exact bool) {
+ return int(t.locale), t.full == nil
+}
+
+// languageTag returns t stripped of regional variant indicators.
+//
+// At the moment this means it is stripped of a regional and variant subtag "rg"
+// and "va" in the "u" extension.
+func (t Tag) languageTag() Tag {
+ if t.full == nil {
+ return Tag{language: t.language, locale: t.language}
+ }
+ tt := t.tag()
+ tt.SetTypeForKey("rg", "")
+ tt.SetTypeForKey("va", "")
+ return makeTag(tt)
+}
+
+// regionalTag returns the regional variant of the tag.
+//
+// At the moment this means that the region is set from the regional subtag
+// "rg" in the "u" extension.
+func (t Tag) regionalTag() Tag {
+ rt := Tag{language: t.locale, locale: t.locale}
+ if t.full == nil {
+ return rt
+ }
+ t, _ = Raw.Compose(rt, t.Variants(), t.Extensions())
+ t, _ = t.SetTypeForKey("rg", "")
+ return t
}
func compactIndex(t language.Tag) (index compactID, exact bool) {
diff --git a/language/language_test.go b/language/language_test.go
index 273f61f..168af38 100644
--- a/language/language_test.go
+++ b/language/language_test.go
@@ -46,35 +46,60 @@
}
}
-func TestCompactIndex(t *testing.T) {
- tests := []struct {
- tag string
- index compactID
- ok bool
- }{
- // TODO: these values will change with each CLDR update. This issue
- // will be solved if we decide to fix the indexes.
- {"und", undIndex, true},
- {"ca-ES-valencia", caESvalenciaIndex, true},
- {"ca-ES-valencia-u-va-posix", caESvalenciaIndex, false},
- {"ca-ES-valencia-u-co-phonebk", caESvalenciaIndex, false},
- {"ca-ES-valencia-u-co-phonebk-va-posix", caESvalenciaIndex, false},
- {"x-klingon", 0, false},
- {"en-US", enUSIndex, true},
- {"en-US-u-va-posix", enUSuvaposixIndex, true},
- {"en", enIndex, true},
- {"en-u-co-phonebk", enIndex, false},
- {"en-001", en001Index, true},
- {"zh-Hant-HK", zhHantHKIndex, true},
- {"zh-HK", zhHantHKIndex, false}, // maximized to zh-Hant-HK
- {"nl-Beng", 0, false}, // parent skips script
- {"nl-NO", nlIndex, false}, // region is ignored
- {"nl-Latn-NO", nlIndex, false},
- {"nl-Latn-NO-u-co-phonebk", nlIndex, false},
- {"nl-Latn-NO-valencia", nlIndex, false},
- {"nl-Latn-NO-oxendict", nlIndex, false},
- {"sh", shIndex, true}, // From plural rules.
+func TestString(t *testing.T) {
+ tests := []string{
+ "no-u-rg-dkzzzz",
}
+ for i, s := range tests {
+ tag := Make(s)
+ if tag.String() != s {
+ t.Errorf("%d:%s: got %s: want %s (%#v)", i, s, tag.String(), s, tag)
+ }
+ }
+}
+
+type compactTest struct {
+ tag string
+ index compactID
+ ok bool
+}
+
+var compactTests = []compactTest{
+ // TODO: these values will change with each CLDR update. This issue
+ // will be solved if we decide to fix the indexes.
+ {"und", undIndex, true},
+ {"ca-ES-valencia", caESvalenciaIndex, true},
+ {"ca-ES-valencia-u-va-posix", caESvalenciaIndex, false},
+ {"ca-ES-valencia-u-co-phonebk", caESvalenciaIndex, false},
+ {"ca-ES-valencia-u-co-phonebk-va-posix", caESvalenciaIndex, false},
+ {"x-klingon", 0, false},
+ {"en-US", enUSIndex, true},
+ {"en-US-u-va-posix", enUSuvaposixIndex, true},
+ {"en", enIndex, true},
+ {"en-u-co-phonebk", enIndex, false},
+ {"en-001", en001Index, true},
+ {"zh-Hant-HK", zhHantHKIndex, true},
+ {"zh-HK", zhHantHKIndex, false}, // maximized to zh-Hant-HK
+ {"nl-Beng", 0, false}, // parent skips script
+ {"nl-NO", nlIndex, false}, // region is ignored
+ {"nl-Latn-NO", nlIndex, false},
+ {"nl-Latn-NO-u-co-phonebk", nlIndex, false},
+ {"nl-Latn-NO-valencia", nlIndex, false},
+ {"nl-Latn-NO-oxendict", nlIndex, false},
+ {"sh", shIndex, true}, // From plural rules.
+}
+
+func TestCompactIndex(t *testing.T) {
+ tests := append(compactTests, []compactTest{
+ {"en-GB", enGBIndex, true},
+ {"en-GB-u-rg-uszzzz", enGBIndex, true},
+ {"en-GB-u-rg-USZZZZ", enGBIndex, true},
+ {"en-GB-u-rg-uszzzz-va-posix", enGBIndex, false},
+ {"en-GB-u-co-phonebk-rg-uszzzz", enGBIndex, false},
+ // Invalid region specifications are ignored.
+ {"en-GB-u-rg-usz-va-posix", enGBIndex, false},
+ {"en-GB-u-co-phonebk-rg-usz", enGBIndex, false},
+ }...)
for _, tt := range tests {
x, ok := CompactIndex(Raw.MustParse(tt.tag))
if compactID(x) != tt.index || ok != tt.ok {
@@ -83,6 +108,27 @@
}
}
+func TestRegionalCompactIndex(t *testing.T) {
+ tests := append(compactTests, []compactTest{
+ {"en-GB", enGBIndex, true},
+ {"en-GB-u-rg-uszzzz", enUSIndex, true},
+ {"en-GB-u-rg-USZZZZ", enUSIndex, true},
+ // TODO: use different exact values for language and regional tag?
+ {"en-GB-u-rg-uszzzz-va-posix", enUSuvaposixIndex, false},
+ {"en-GB-u-co-phonebk-rg-uszzzz-va-posix", enUSuvaposixIndex, false},
+ {"en-GB-u-co-phonebk-rg-uszzzz", enUSIndex, false},
+ // Invalid region specifications are ignored.
+ {"en-GB-u-rg-usz-va-posix", enGBIndex, false},
+ {"en-GB-u-co-phonebk-rg-usz", enGBIndex, false},
+ }...)
+ for _, tt := range tests {
+ x, ok := regionalCompactIndex(Raw.MustParse(tt.tag))
+ if compactID(x) != tt.index || ok != tt.ok {
+ t.Errorf("%s: got %d, %v; want %d %v", tt.tag, x, ok, tt.index, tt.ok)
+ }
+ }
+}
+
func TestMarshal(t *testing.T) {
testCases := []string{
// TODO: these values will change with each CLDR update. This issue
@@ -99,6 +145,12 @@
"en-u-co-phonebk",
"en-001",
"sh",
+
+ "en-GB-u-rg-uszzzz",
+ "en-GB-u-rg-uszzzz-va-posix",
+ "en-GB-u-co-phonebk-rg-uszzzz",
+ // Invalid tags should also roundtrip.
+ "en-GB-u-co-phonebk-rg-uszz",
}
for _, tc := range testCases {
var tag Tag
@@ -532,6 +584,16 @@
{"und-Qaai", "und-Zinh", DeprecatedScript},
{"und-Qaai", "und-Qaai", DeprecatedBase},
{"drh", "mn", All}, // drh -> khk -> mn
+
+ {"en-GB-u-rg-uszzzz", "en-GB-u-rg-uszzzz", Raw},
+ {"en-GB-u-rg-USZZZZ", "en-GB-u-rg-uszzzz", Raw},
+ // TODO: use different exact values for language and regional tag?
+ {"en-GB-u-rg-uszzzz-va-posix", "en-GB-u-rg-uszzzz-va-posix", Raw},
+ {"en-GB-u-rg-uszzzz-co-phonebk", "en-GB-u-co-phonebk-rg-uszzzz", Raw},
+ // Invalid region specifications are left as is.
+ {"en-GB-u-rg-usz", "en-GB-u-rg-usz", Raw},
+ {"en-GB-u-rg-usz-va-posix", "en-GB-u-rg-usz-va-posix", Raw},
+ {"en-GB-u-rg-usz-co-phonebk", "en-GB-u-co-phonebk-rg-usz", Raw},
}
for i, tt := range tests {
in, _ := Raw.Parse(tt.in)
@@ -558,6 +620,8 @@
{"co", "en-u-co-phonebk", "phonebk"},
{"co", "en-u-co-phonebk-cu-aud", "phonebk"},
{"co", "x-foo-u-co-phonebk", ""},
+ {"va", "en-US-u-va-posix", "posix"},
+ {"rg", "en-u-rg-gbzzzz", "gbzzzz"},
{"nu", "en-u-co-phonebk-nu-arabic", "arabic"},
{"kc", "cmn-u-co-stroke", ""},
}
@@ -656,6 +720,19 @@
{"pt-MZ", "pt-PT"},
{"pt-ST", "pt-PT"},
{"pt-TL", "pt-PT"},
+
+ {"en-GB-u-co-phonebk-rg-uszzzz", "en-GB"},
+ {"en-GB-u-rg-uszzzz", "en-GB"},
+ {"en-US-u-va-posix", "en-US"},
+
+ // Difference between language and regional tag.
+ {"ca-ES-valencia", "ca-ES"},
+ {"ca-ES-valencia-u-rg-ptzzzz", "ca-ES"},
+ {"en-US-u-va-variant", "en-US"},
+ {"en-u-va-variant", "en"},
+ {"en-u-rg-gbzzzz", "en"},
+ {"en-US-u-rg-gbzzzz", "en-US"},
+ {"nl-US-u-rg-gbzzzz", "nl-US"},
}
for _, tt := range tests {
tag := Raw.MustParse(tt.in)
diff --git a/language/match.go b/language/match.go
index d84e972..0edbc9d 100644
--- a/language/match.go
+++ b/language/match.go
@@ -6,6 +6,7 @@
import (
"errors"
+ "strings"
"golang.org/x/text/language/internal"
)
@@ -104,9 +105,14 @@
}
// TODO: select first language tag based on script.
}
- if w.RegionID != 0 && tt.RegionID != 0 && tt.RegionID.Contains(w.RegionID) {
- tt.RegionID = w.RegionID
- tt.RemakeString()
+ if w.RegionID != tt.RegionID && w.RegionID != 0 {
+ if w.RegionID != 0 && tt.RegionID != 0 && tt.RegionID.Contains(w.RegionID) {
+ tt.RegionID = w.RegionID
+ tt.RemakeString()
+ } else if r := w.RegionID.String(); len(r) == 2 {
+ // TODO: also filter macro and deprecated.
+ tt, _ = tt.SetTypeForKey("rg", strings.ToLower(r)+"zzzz")
+ }
}
// Copy options from the user-provided tag into the result tag. This is hard
// to do after the fact, so we do it here.
diff --git a/language/match_test.go b/language/match_test.go
index 3bfefff..c21b863 100644
--- a/language/match_test.go
+++ b/language/match_test.go
@@ -40,13 +40,15 @@
gotCombined, index, conf := NewMatcher(supported).Match(desired...)
gotMatch := supported[index]
- wantMatch := mk(p.String(2))
+ wantMatch := Raw.Make(p.String(2)) // wantMatch may be null
if gotMatch != wantMatch {
t.Fatalf("match: got %q; want %q (%v)", gotMatch, wantMatch, conf)
}
- wantCombined, err := Raw.Parse(p.String(3))
- if err == nil && gotCombined != wantCombined {
- t.Errorf("combined: got %q; want %q (%v)", gotCombined, wantCombined, conf)
+ if tag := strings.TrimSpace(p.String(3)); tag != "" {
+ wantCombined := Raw.MustParse(tag)
+ if err == nil && gotCombined != wantCombined {
+ t.Errorf("combined: got %q; want %q (%v)", gotCombined, wantCombined, conf)
+ }
}
})
})
@@ -68,7 +70,7 @@
// which is better.
// Inconsistencies in combined. I think the Go approach is more appropriate.
- // We could use -u-rg- and -u-va- as alternative.
+ // We could use -u-rg- as alternative.
"und,fr/fr-BE-fonipa": true, // combined: got "fr"; want "fr-BE-fonipa"
"und,fr-CA/fr-BE-fonipa": true, // combined: got "fr-CA"; want "fr-BE-fonipa"
"und,fr-fonupa/fr-BE-fonipa": true, // combined: got "fr-fonupa"; want "fr-BE-fonipa"
diff --git a/language/parse.go b/language/parse.go
index e54a0a4..f0e0b64 100644
--- a/language/parse.go
+++ b/language/parse.go
@@ -74,10 +74,10 @@
// than once, the latter will overwrite the former. Variants and Extensions are
// accumulated, but if two extensions of the same type are passed, the latter
// will replace the former. For -u extensions, though, the key-type pairs are
-// added, where later values overwrite older ones. A Tag overwrites all former values and typically
-// only makes sense as the first argument. The resulting tag is returned after
-// canonicalizing using CanonType c. If one or more errors are encountered,
-// one of the errors is returned.
+// added, where later values overwrite older ones. A Tag overwrites all former
+// values and typically only makes sense as the first argument. The resulting
+// tag is returned after canonicalizing using CanonType c. If one or more errors
+// are encountered, one of the errors is returned.
func (c CanonType) Compose(part ...interface{}) (t Tag, err error) {
var b language.Builder
if err = update(&b, part...); err != nil {
diff --git a/language/parse_test.go b/language/parse_test.go
index 7a5b54b..3ac1642 100644
--- a/language/parse_test.go
+++ b/language/parse_test.go
@@ -36,6 +36,11 @@
{in: "root", lang: "und"},
{in: "und", lang: "und"},
{in: "en", lang: "en"},
+
+ {in: "en-US-u-va-posix", lang: "en", region: "US", ext: "u-va-posix"},
+ {in: "ca-ES-valencia", lang: "ca", region: "ES", variants: "valencia"},
+ {in: "en-US-u-rg-gbzzzz", lang: "en", region: "US", ext: "u-rg-gbzzzz"},
+
{in: "xy", lang: "und", invalid: true},
{in: "en-ZY", lang: "en", invalid: true},
{in: "gsw", lang: "gsw"},
diff --git a/language/testdata/GoLocaleMatcherTest.txt b/language/testdata/GoLocaleMatcherTest.txt
index 4f4c609..32a649f 100644
--- a/language/testdata/GoLocaleMatcherTest.txt
+++ b/language/testdata/GoLocaleMatcherTest.txt
@@ -9,7 +9,7 @@
zh-CN, zh-TW, iw ; zh ; zh-CN
zh-CN, zh-TW, iw ; zh-Hans-CN ; zh-CN
zh-CN, zh-TW, iw ; zh-Hant-HK ; zh-TW
-zh-CN, zh-TW, iw ; he-IT ; iw ; iw
+zh-CN, zh-TW, iw ; he-IT ; iw ; iw-u-rg-itzzzz
# language-specific script fallbacks 1
en, sr, nl ; sr-Latn ; sr
@@ -125,7 +125,7 @@
pt, pt-PT ; pt-ES ; pt-PT
# if no preferred locale specified, pick top language, not regional
-en, fr, fr-CA, fr-CH ; fr-US ; fr #TODO: ; fr-u-rg-US
+en, fr, fr-CA, fr-CH ; fr-US ; fr ; fr-u-rg-uszzzz
# region distance German
de-AT, de-DE, de-CH ; de ; de-DE
@@ -218,9 +218,14 @@
en, en-US, en-GB, es, es-419, pt, pt-BR, pt-PT, zh, zh-Hant, zh-Hant-HK ; pt-TL ; pt-PT
# preserve extensions
-en, de, sl-nedis ; de-FR-u-co-phonebk ; de ; de-u-co-phonebk
+en, de, sl-nedis ; de-FR-u-co-phonebk ; de ; de-u-co-phonebk-rg-frzzzz
en, de, sl-nedis ; sl-nedis-u-cu-eur ; sl-nedis ; sl-nedis-u-cu-eur
en, de, sl-nedis ; sl-u-cu-eur ; sl-nedis ; sl-nedis-u-cu-eur
-en, de, sl-nedis ; sl-HR-nedis-u-cu-eur ; sl-nedis ; sl-nedis-u-cu-eur
+en, de, sl-nedis ; sl-HR-nedis-u-cu-eur ; sl-nedis ; sl-nedis-u-cu-eur-rg-hrzzzz
en, de, sl-nedis ; de-t-m0-iso-i0-pinyin ; de ; de-t-m0-iso-i0-pinyin
+und, nl ; nl-BE-fonipa ; nl ; nl-u-rg-bezzzz
+und, nl-CA ; nl-BE-fonipa ; nl-CA ; nl-CA-u-rg-bezzzz
+und, nl-fonupa ; nl-BE-fonipa ; nl-fonupa ; nl-fonupa-u-rg-bezzzz
+und, no ; nn-DK-fonipa ; no ; no-u-rg-dkzzzz
+und, en-GB-u-sd-usca ; en-US-fonipa-u-nu-Arab-ca-buddhist-sd-usdc-t-m0-iso-i0-pinyin ; en-GB-u-sd-usca ; en-GB-t-m0-iso-i0-pinyin-u-ca-buddhist-nu-Arab-rg-uszzzz-sd-usca
\ No newline at end of file
diff --git a/message/catalog/catalog_test.go b/message/catalog/catalog_test.go
index 08bfdc7..3de4c52 100644
--- a/message/catalog/catalog_test.go
+++ b/message/catalog/catalog_test.go
@@ -63,7 +63,7 @@
},
match: []string{
"gr -> en",
- "en-US -> en",
+ "en-US -> en-u-rg-uszzzz",
},
tags: langs("en"),
}, {
diff --git a/message/catalog_test.go b/message/catalog_test.go
index 7a2301c..ff409a8 100644
--- a/message/catalog_test.go
+++ b/message/catalog_test.go
@@ -23,7 +23,7 @@
want string
}{{
args: "de-CH",
- want: "de",
+ want: "de-u-rg-chzzzz",
}, {
args: "bn-u-nu-latn|en-US,en;q=0.9,de;q=0.8,nl;q=0.7",
want: "bn-u-nu-latn",
