| // Copyright 2013 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| //go:generate go run gen.go -output tables.go |
| |
| package language |
| |
| // TODO: Remove above NOTE after: |
| // - verifying that tables are dropped correctly (most notably matcher tables). |
| |
| import ( |
| "strings" |
| |
| "golang.org/x/text/internal/language" |
| "golang.org/x/text/internal/language/compact" |
| ) |
| |
| // Tag represents a BCP 47 language tag. It is used to specify an instance of a |
| // specific language or locale. All language tag values are guaranteed to be |
| // well-formed. |
| type Tag compact.Tag |
| |
| func makeTag(t language.Tag) (tag Tag) { |
| return Tag(compact.Make(t)) |
| } |
| |
| func (t *Tag) tag() language.Tag { |
| return (*compact.Tag)(t).Tag() |
| } |
| |
| func (t *Tag) isCompact() bool { |
| return (*compact.Tag)(t).IsCompact() |
| } |
| |
| // TODO: improve performance. |
| func (t *Tag) lang() language.Language { return t.tag().LangID } |
| func (t *Tag) region() language.Region { return t.tag().RegionID } |
| func (t *Tag) script() language.Script { return t.tag().ScriptID } |
| |
| // Make is a convenience wrapper for Parse that omits the error. |
| // In case of an error, a sensible default is returned. |
| func Make(s string) Tag { |
| return Default.Make(s) |
| } |
| |
| // Make is a convenience wrapper for c.Parse that omits the error. |
| // In case of an error, a sensible default is returned. |
| func (c CanonType) Make(s string) Tag { |
| t, _ := c.Parse(s) |
| return t |
| } |
| |
| // Raw returns the raw base language, script and region, without making an |
| // attempt to infer their values. |
| func (t Tag) Raw() (b Base, s Script, r Region) { |
| tt := t.tag() |
| return Base{tt.LangID}, Script{tt.ScriptID}, Region{tt.RegionID} |
| } |
| |
| // IsRoot returns true if t is equal to language "und". |
| func (t Tag) IsRoot() bool { |
| return compact.Tag(t).IsRoot() |
| } |
| |
| // CanonType can be used to enable or disable various types of canonicalization. |
| type CanonType int |
| |
| const ( |
| // Replace deprecated base languages with their preferred replacements. |
| DeprecatedBase CanonType = 1 << iota |
| // Replace deprecated scripts with their preferred replacements. |
| DeprecatedScript |
| // Replace deprecated regions with their preferred replacements. |
| DeprecatedRegion |
| // Remove redundant scripts. |
| SuppressScript |
| // Normalize legacy encodings. This includes legacy languages defined in |
| // CLDR as well as bibliographic codes defined in ISO-639. |
| Legacy |
| // Map the dominant language of a macro language group to the macro language |
| // subtag. For example cmn -> zh. |
| Macro |
| // The CLDR flag should be used if full compatibility with CLDR is required. |
| // There are a few cases where language.Tag may differ from CLDR. To follow all |
| // of CLDR's suggestions, use All|CLDR. |
| CLDR |
| |
| // Raw can be used to Compose or Parse without Canonicalization. |
| Raw CanonType = 0 |
| |
| // Replace all deprecated tags with their preferred replacements. |
| Deprecated = DeprecatedBase | DeprecatedScript | DeprecatedRegion |
| |
| // All canonicalizations recommended by BCP 47. |
| BCP47 = Deprecated | SuppressScript |
| |
| // All canonicalizations. |
| All = BCP47 | Legacy | Macro |
| |
| // Default is the canonicalization used by Parse, Make and Compose. To |
| // preserve as much information as possible, canonicalizations that remove |
| // potentially valuable information are not included. The Matcher is |
| // designed to recognize similar tags that would be the same if |
| // they were canonicalized using All. |
| Default = Deprecated | Legacy |
| |
| canonLang = DeprecatedBase | Legacy | Macro |
| |
| // TODO: LikelyScript, LikelyRegion: suppress similar to ICU. |
| ) |
| |
| // canonicalize returns the canonicalized equivalent of the tag and |
| // whether there was any change. |
| func canonicalize(c CanonType, t language.Tag) (language.Tag, bool) { |
| if c == Raw { |
| return t, false |
| } |
| changed := false |
| if c&SuppressScript != 0 { |
| if t.LangID.SuppressScript() == t.ScriptID { |
| t.ScriptID = 0 |
| changed = true |
| } |
| } |
| if c&canonLang != 0 { |
| for { |
| if l, aliasType := t.LangID.Canonicalize(); l != t.LangID { |
| switch aliasType { |
| case language.Legacy: |
| if c&Legacy != 0 { |
| if t.LangID == _sh && t.ScriptID == 0 { |
| t.ScriptID = _Latn |
| } |
| t.LangID = l |
| changed = true |
| } |
| case language.Macro: |
| if c&Macro != 0 { |
| // We deviate here from CLDR. The mapping "nb" -> "no" |
| // qualifies as a typical Macro language mapping. However, |
| // for legacy reasons, CLDR maps "no", the macro language |
| // code for Norwegian, to the dominant variant "nb". This |
| // change is currently under consideration for CLDR as well. |
| // See https://unicode.org/cldr/trac/ticket/2698 and also |
| // https://unicode.org/cldr/trac/ticket/1790 for some of the |
| // practical implications. TODO: this check could be removed |
| // if CLDR adopts this change. |
| if c&CLDR == 0 || t.LangID != _nb { |
| changed = true |
| t.LangID = l |
| } |
| } |
| case language.Deprecated: |
| if c&DeprecatedBase != 0 { |
| if t.LangID == _mo && t.RegionID == 0 { |
| t.RegionID = _MD |
| } |
| t.LangID = l |
| changed = true |
| // Other canonicalization types may still apply. |
| continue |
| } |
| } |
| } else if c&Legacy != 0 && t.LangID == _no && c&CLDR != 0 { |
| t.LangID = _nb |
| changed = true |
| } |
| break |
| } |
| } |
| if c&DeprecatedScript != 0 { |
| if t.ScriptID == _Qaai { |
| changed = true |
| t.ScriptID = _Zinh |
| } |
| } |
| if c&DeprecatedRegion != 0 { |
| if r := t.RegionID.Canonicalize(); r != t.RegionID { |
| changed = true |
| t.RegionID = r |
| } |
| } |
| return t, changed |
| } |
| |
| // Canonicalize returns the canonicalized equivalent of the tag. |
| func (c CanonType) Canonicalize(t Tag) (Tag, error) { |
| // First try fast path. |
| if t.isCompact() { |
| if _, changed := canonicalize(c, compact.Tag(t).Tag()); !changed { |
| return t, nil |
| } |
| } |
| // It is unlikely that one will canonicalize a tag after matching. So do |
| // a slow but simple approach here. |
| if tag, changed := canonicalize(c, t.tag()); changed { |
| tag.RemakeString() |
| return makeTag(tag), nil |
| } |
| return t, nil |
| |
| } |
| |
| // Confidence indicates the level of certainty for a given return value. |
| // For example, Serbian may be written in Cyrillic or Latin script. |
| // The confidence level indicates whether a value was explicitly specified, |
| // whether it is typically the only possible value, or whether there is |
| // an ambiguity. |
| type Confidence int |
| |
| const ( |
| No Confidence = iota // full confidence that there was no match |
| Low // most likely value picked out of a set of alternatives |
| High // value is generally assumed to be the correct match |
| Exact // exact match or explicitly specified value |
| ) |
| |
| var confName = []string{"No", "Low", "High", "Exact"} |
| |
| func (c Confidence) String() string { |
| return confName[c] |
| } |
| |
| // String returns the canonical string representation of the language tag. |
| func (t Tag) String() string { |
| return t.tag().String() |
| } |
| |
| // MarshalText implements encoding.TextMarshaler. |
| func (t Tag) MarshalText() (text []byte, err error) { |
| return t.tag().MarshalText() |
| } |
| |
| // UnmarshalText implements encoding.TextUnmarshaler. |
| func (t *Tag) UnmarshalText(text []byte) error { |
| var tag language.Tag |
| err := tag.UnmarshalText(text) |
| *t = makeTag(tag) |
| return err |
| } |
| |
| // Base returns the base language of the language tag. If the base language is |
| // unspecified, an attempt will be made to infer it from the context. |
| // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change. |
| func (t Tag) Base() (Base, Confidence) { |
| if b := t.lang(); b != 0 { |
| return Base{b}, Exact |
| } |
| tt := t.tag() |
| c := High |
| if tt.ScriptID == 0 && !tt.RegionID.IsCountry() { |
| c = Low |
| } |
| if tag, err := tt.Maximize(); err == nil && tag.LangID != 0 { |
| return Base{tag.LangID}, c |
| } |
| return Base{0}, No |
| } |
| |
| // Script infers the script for the language tag. If it was not explicitly given, it will infer |
| // a most likely candidate. |
| // If more than one script is commonly used for a language, the most likely one |
| // is returned with a low confidence indication. For example, it returns (Cyrl, Low) |
| // for Serbian. |
| // If a script cannot be inferred (Zzzz, No) is returned. We do not use Zyyy (undetermined) |
| // as one would suspect from the IANA registry for BCP 47. In a Unicode context Zyyy marks |
| // common characters (like 1, 2, 3, '.', etc.) and is therefore more like multiple scripts. |
| // See https://www.unicode.org/reports/tr24/#Values for more details. Zzzz is also used for |
| // unknown value in CLDR. (Zzzz, Exact) is returned if Zzzz was explicitly specified. |
| // Note that an inferred script is never guaranteed to be the correct one. Latin is |
| // almost exclusively used for Afrikaans, but Arabic has been used for some texts |
| // in the past. Also, the script that is commonly used may change over time. |
| // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change. |
| func (t Tag) Script() (Script, Confidence) { |
| if scr := t.script(); scr != 0 { |
| return Script{scr}, Exact |
| } |
| tt := t.tag() |
| sc, c := language.Script(_Zzzz), No |
| if scr := tt.LangID.SuppressScript(); scr != 0 { |
| // Note: it is not always the case that a language with a suppress |
| // script value is only written in one script (e.g. kk, ms, pa). |
| if tt.RegionID == 0 { |
| return Script{scr}, High |
| } |
| sc, c = scr, High |
| } |
| if tag, err := tt.Maximize(); err == nil { |
| if tag.ScriptID != sc { |
| sc, c = tag.ScriptID, Low |
| } |
| } else { |
| tt, _ = canonicalize(Deprecated|Macro, tt) |
| if tag, err := tt.Maximize(); err == nil && tag.ScriptID != sc { |
| sc, c = tag.ScriptID, Low |
| } |
| } |
| return Script{sc}, c |
| } |
| |
| // Region returns the region for the language tag. If it was not explicitly given, it will |
| // infer a most likely candidate from the context. |
| // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change. |
| func (t Tag) Region() (Region, Confidence) { |
| if r := t.region(); r != 0 { |
| return Region{r}, Exact |
| } |
| tt := t.tag() |
| if tt, err := tt.Maximize(); err == nil { |
| return Region{tt.RegionID}, Low // TODO: differentiate between high and low. |
| } |
| tt, _ = canonicalize(Deprecated|Macro, tt) |
| if tag, err := tt.Maximize(); err == nil { |
| return Region{tag.RegionID}, Low |
| } |
| return Region{_ZZ}, No // TODO: return world instead of undetermined? |
| } |
| |
| // Variants returns the variants specified explicitly for this language tag. |
| // or nil if no variant was specified. |
| func (t Tag) Variants() []Variant { |
| if !compact.Tag(t).MayHaveVariants() { |
| return nil |
| } |
| v := []Variant{} |
| x, str := "", t.tag().Variants() |
| for str != "" { |
| x, str = nextToken(str) |
| v = append(v, Variant{x}) |
| } |
| return v |
| } |
| |
| // Parent returns the CLDR parent of t. In CLDR, missing fields in data for a |
| // specific language are substituted with fields from the parent language. |
| // The parent for a language may change for newer versions of CLDR. |
| // |
| // Parent returns a tag for a less specific language that is mutually |
| // intelligible or Und if there is no such language. This may not be the same as |
| // simply stripping the last BCP 47 subtag. For instance, the parent of "zh-TW" |
| // is "zh-Hant", and the parent of "zh-Hant" is "und". |
| func (t Tag) Parent() Tag { |
| return Tag(compact.Tag(t).Parent()) |
| } |
| |
| // nextToken returns token t and the rest of the string. |
| func nextToken(s string) (t, tail string) { |
| p := strings.Index(s[1:], "-") |
| if p == -1 { |
| return s[1:], "" |
| } |
| p++ |
| return s[1:p], s[p:] |
| } |
| |
| // Extension is a single BCP 47 extension. |
| type Extension struct { |
| s string |
| } |
| |
| // String returns the string representation of the extension, including the |
| // type tag. |
| func (e Extension) String() string { |
| return e.s |
| } |
| |
| // ParseExtension parses s as an extension and returns it on success. |
| func ParseExtension(s string) (e Extension, err error) { |
| ext, err := language.ParseExtension(s) |
| return Extension{ext}, err |
| } |
| |
| // Type returns the one-byte extension type of e. It returns 0 for the zero |
| // exception. |
| func (e Extension) Type() byte { |
| if e.s == "" { |
| return 0 |
| } |
| return e.s[0] |
| } |
| |
| // Tokens returns the list of tokens of e. |
| func (e Extension) Tokens() []string { |
| return strings.Split(e.s, "-") |
| } |
| |
| // Extension returns the extension of type x for tag t. It will return |
| // false for ok if t does not have the requested extension. The returned |
| // extension will be invalid in this case. |
| func (t Tag) Extension(x byte) (ext Extension, ok bool) { |
| if !compact.Tag(t).MayHaveExtensions() { |
| return Extension{}, false |
| } |
| e, ok := t.tag().Extension(x) |
| return Extension{e}, ok |
| } |
| |
| // Extensions returns all extensions of t. |
| func (t Tag) Extensions() []Extension { |
| if !compact.Tag(t).MayHaveExtensions() { |
| return nil |
| } |
| e := []Extension{} |
| for _, ext := range t.tag().Extensions() { |
| e = append(e, Extension{ext}) |
| } |
| return e |
| } |
| |
| // TypeForKey returns the type associated with the given key, where key and type |
| // are of the allowed values defined for the Unicode locale extension ('u') in |
| // https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers. |
| // TypeForKey will traverse the inheritance chain to get the correct value. |
| // |
| // If there are multiple types associated with a key, only the first will be |
| // returned. If there is no type associated with a key, it returns the empty |
| // string. |
| func (t Tag) TypeForKey(key string) string { |
| if !compact.Tag(t).MayHaveExtensions() { |
| if key != "rg" && key != "va" { |
| return "" |
| } |
| } |
| return t.tag().TypeForKey(key) |
| } |
| |
| // SetTypeForKey returns a new Tag with the key set to type, where key and type |
| // are of the allowed values defined for the Unicode locale extension ('u') in |
| // https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers. |
| // An empty value removes an existing pair with the same key. |
| func (t Tag) SetTypeForKey(key, value string) (Tag, error) { |
| tt, err := t.tag().SetTypeForKey(key, value) |
| return makeTag(tt), err |
| } |
| |
| // NumCompactTags is the number of compact tags. The maximum tag is |
| // NumCompactTags-1. |
| const NumCompactTags = compact.NumCompactTags |
| |
| // CompactIndex returns an index, where 0 <= index < NumCompactTags, for tags |
| // for which data exists in the text repository.The index will change over time |
| // and should not be stored in persistent storage. If t does not match a compact |
| // index, exact will be false and the compact index will be returned for the |
| // first match after repeatedly taking the Parent of t. |
| func CompactIndex(t Tag) (index int, exact bool) { |
| id, exact := compact.LanguageID(compact.Tag(t)) |
| return int(id), exact |
| } |
| |
| var root = language.Tag{} |
| |
| // Base is an ISO 639 language code, used for encoding the base language |
| // of a language tag. |
| type Base struct { |
| langID language.Language |
| } |
| |
| // ParseBase parses a 2- or 3-letter ISO 639 code. |
| // It returns a ValueError if s is a well-formed but unknown language identifier |
| // or another error if another error occurred. |
| func ParseBase(s string) (Base, error) { |
| l, err := language.ParseBase(s) |
| return Base{l}, err |
| } |
| |
| // String returns the BCP 47 representation of the base language. |
| func (b Base) String() string { |
| return b.langID.String() |
| } |
| |
| // ISO3 returns the ISO 639-3 language code. |
| func (b Base) ISO3() string { |
| return b.langID.ISO3() |
| } |
| |
| // IsPrivateUse reports whether this language code is reserved for private use. |
| func (b Base) IsPrivateUse() bool { |
| return b.langID.IsPrivateUse() |
| } |
| |
| // Script is a 4-letter ISO 15924 code for representing scripts. |
| // It is idiomatically represented in title case. |
| type Script struct { |
| scriptID language.Script |
| } |
| |
| // ParseScript parses a 4-letter ISO 15924 code. |
| // It returns a ValueError if s is a well-formed but unknown script identifier |
| // or another error if another error occurred. |
| func ParseScript(s string) (Script, error) { |
| sc, err := language.ParseScript(s) |
| return Script{sc}, err |
| } |
| |
| // String returns the script code in title case. |
| // It returns "Zzzz" for an unspecified script. |
| func (s Script) String() string { |
| return s.scriptID.String() |
| } |
| |
| // IsPrivateUse reports whether this script code is reserved for private use. |
| func (s Script) IsPrivateUse() bool { |
| return s.scriptID.IsPrivateUse() |
| } |
| |
| // Region is an ISO 3166-1 or UN M.49 code for representing countries and regions. |
| type Region struct { |
| regionID language.Region |
| } |
| |
| // EncodeM49 returns the Region for the given UN M.49 code. |
| // It returns an error if r is not a valid code. |
| func EncodeM49(r int) (Region, error) { |
| rid, err := language.EncodeM49(r) |
| return Region{rid}, err |
| } |
| |
| // ParseRegion parses a 2- or 3-letter ISO 3166-1 or a UN M.49 code. |
| // It returns a ValueError if s is a well-formed but unknown region identifier |
| // or another error if another error occurred. |
| func ParseRegion(s string) (Region, error) { |
| r, err := language.ParseRegion(s) |
| return Region{r}, err |
| } |
| |
| // String returns the BCP 47 representation for the region. |
| // It returns "ZZ" for an unspecified region. |
| func (r Region) String() string { |
| return r.regionID.String() |
| } |
| |
| // ISO3 returns the 3-letter ISO code of r. |
| // Note that not all regions have a 3-letter ISO code. |
| // In such cases this method returns "ZZZ". |
| func (r Region) ISO3() string { |
| return r.regionID.ISO3() |
| } |
| |
| // M49 returns the UN M.49 encoding of r, or 0 if this encoding |
| // is not defined for r. |
| func (r Region) M49() int { |
| return r.regionID.M49() |
| } |
| |
| // IsPrivateUse reports whether r has the ISO 3166 User-assigned status. This |
| // may include private-use tags that are assigned by CLDR and used in this |
| // implementation. So IsPrivateUse and IsCountry can be simultaneously true. |
| func (r Region) IsPrivateUse() bool { |
| return r.regionID.IsPrivateUse() |
| } |
| |
| // IsCountry returns whether this region is a country or autonomous area. This |
| // includes non-standard definitions from CLDR. |
| func (r Region) IsCountry() bool { |
| return r.regionID.IsCountry() |
| } |
| |
| // IsGroup returns whether this region defines a collection of regions. This |
| // includes non-standard definitions from CLDR. |
| func (r Region) IsGroup() bool { |
| return r.regionID.IsGroup() |
| } |
| |
| // Contains returns whether Region c is contained by Region r. It returns true |
| // if c == r. |
| func (r Region) Contains(c Region) bool { |
| return r.regionID.Contains(c.regionID) |
| } |
| |
| // TLD returns the country code top-level domain (ccTLD). UK is returned for GB. |
| // In all other cases it returns either the region itself or an error. |
| // |
| // This method may return an error for a region for which there exists a |
| // canonical form with a ccTLD. To get that ccTLD canonicalize r first. The |
| // region will already be canonicalized it was obtained from a Tag that was |
| // obtained using any of the default methods. |
| func (r Region) TLD() (Region, error) { |
| tld, err := r.regionID.TLD() |
| return Region{tld}, err |
| } |
| |
| // Canonicalize returns the region or a possible replacement if the region is |
| // deprecated. It will not return a replacement for deprecated regions that |
| // are split into multiple regions. |
| func (r Region) Canonicalize() Region { |
| return Region{r.regionID.Canonicalize()} |
| } |
| |
| // Variant represents a registered variant of a language as defined by BCP 47. |
| type Variant struct { |
| variant string |
| } |
| |
| // ParseVariant parses and returns a Variant. An error is returned if s is not |
| // a valid variant. |
| func ParseVariant(s string) (Variant, error) { |
| v, err := language.ParseVariant(s) |
| return Variant{v.String()}, err |
| } |
| |
| // String returns the string representation of the variant. |
| func (v Variant) String() string { |
| return v.variant |
| } |