language/language.go - text - Git at Google

 // Copyright 2013 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 //go:generate go run gen.go -output tables.go

 package language

 // TODO: Remove above NOTE after:
 // - verifying that tables are dropped correctly (most notably matcher tables).

 import (
 	"strings"

 	"golang.org/x/text/internal/language"
 	"golang.org/x/text/internal/language/compact"
 )

 // Tag represents a BCP 47 language tag. It is used to specify an instance of a
 // specific language or locale. All language tag values are guaranteed to be
 // well-formed.
 type Tag compact.Tag

 func makeTag(t language.Tag) (tag Tag) {
 	return Tag(compact.Make(t))
 }

 func (t *Tag) tag() language.Tag {
 	return (*compact.Tag)(t).Tag()
 }

 func (t *Tag) isCompact() bool {
 	return (*compact.Tag)(t).IsCompact()
 }

 // TODO: improve performance.
 func (t *Tag) lang() language.Language { return t.tag().LangID }
 func (t *Tag) region() language.Region { return t.tag().RegionID }
 func (t *Tag) script() language.Script { return t.tag().ScriptID }

 // Make is a convenience wrapper for Parse that omits the error.
 // In case of an error, a sensible default is returned.
 func Make(s string) Tag {
 	return Default.Make(s)
 }

 // Make is a convenience wrapper for c.Parse that omits the error.
 // In case of an error, a sensible default is returned.
 func (c CanonType) Make(s string) Tag {
 	t, _ := c.Parse(s)
 	return t
 }

 // Raw returns the raw base language, script and region, without making an
 // attempt to infer their values.
 func (t Tag) Raw() (b Base, s Script, r Region) {
 	tt := t.tag()
 	return Base{tt.LangID}, Script{tt.ScriptID}, Region{tt.RegionID}
 }

 // IsRoot returns true if t is equal to language "und".
 func (t Tag) IsRoot() bool {
 	return compact.Tag(t).IsRoot()
 }

 // CanonType can be used to enable or disable various types of canonicalization.
 type CanonType int

 const (
 	// Replace deprecated base languages with their preferred replacements.
 	DeprecatedBase CanonType = 1 << iota
 	// Replace deprecated scripts with their preferred replacements.
 	DeprecatedScript
 	// Replace deprecated regions with their preferred replacements.
 	DeprecatedRegion
 	// Remove redundant scripts.
 	SuppressScript
 	// Normalize legacy encodings. This includes legacy languages defined in
 	// CLDR as well as bibliographic codes defined in ISO-639.
 	Legacy
 	// Map the dominant language of a macro language group to the macro language
 	// subtag. For example cmn -> zh.
 	Macro
 	// The CLDR flag should be used if full compatibility with CLDR is required.
 	// There are a few cases where language.Tag may differ from CLDR. To follow all
 	// of CLDR's suggestions, use All|CLDR.
 	CLDR

 	// Raw can be used to Compose or Parse without Canonicalization.
 	Raw CanonType = 0

 	// Replace all deprecated tags with their preferred replacements.
 	Deprecated = DeprecatedBase | DeprecatedScript | DeprecatedRegion

 	// All canonicalizations recommended by BCP 47.
 	BCP47 = Deprecated | SuppressScript

 	// All canonicalizations.
 	All = BCP47 | Legacy | Macro

 	// Default is the canonicalization used by Parse, Make and Compose. To
 	// preserve as much information as possible, canonicalizations that remove
 	// potentially valuable information are not included. The Matcher is
 	// designed to recognize similar tags that would be the same if
 	// they were canonicalized using All.
 	Default = Deprecated | Legacy

 	canonLang = DeprecatedBase | Legacy | Macro

 	// TODO: LikelyScript, LikelyRegion: suppress similar to ICU.
 )

 // canonicalize returns the canonicalized equivalent of the tag and
 // whether there was any change.
 func canonicalize(c CanonType, t language.Tag) (language.Tag, bool) {
 	if c == Raw {
 		return t, false
 	}
 	changed := false
 	if c&SuppressScript != 0 {
 		if t.LangID.SuppressScript() == t.ScriptID {
 			t.ScriptID = 0
 			changed = true
 		}
 	}
 	if c&canonLang != 0 {
 		for {
 			if l, aliasType := t.LangID.Canonicalize(); l != t.LangID {
 				switch aliasType {
 				case language.Legacy:
 					if c&Legacy != 0 {
 						if t.LangID == _sh && t.ScriptID == 0 {
 							t.ScriptID = _Latn
 						}
 						t.LangID = l
 						changed = true
 					}
 				case language.Macro:
 					if c&Macro != 0 {
 						// We deviate here from CLDR. The mapping "nb" -> "no"
 						// qualifies as a typical Macro language mapping.  However,
 						// for legacy reasons, CLDR maps "no", the macro language
 						// code for Norwegian, to the dominant variant "nb". This
 						// change is currently under consideration for CLDR as well.
 						// See https://unicode.org/cldr/trac/ticket/2698 and also
 						// https://unicode.org/cldr/trac/ticket/1790 for some of the
 						// practical implications. TODO: this check could be removed
 						// if CLDR adopts this change.
 						if c&CLDR == 0 || t.LangID != _nb {
 							changed = true
 							t.LangID = l
 						}
 					}
 				case language.Deprecated:
 					if c&DeprecatedBase != 0 {
 						if t.LangID == _mo && t.RegionID == 0 {
 							t.RegionID = _MD
 						}
 						t.LangID = l
 						changed = true
 						// Other canonicalization types may still apply.
 						continue
 					}
 				}
 			} else if c&Legacy != 0 && t.LangID == _no && c&CLDR != 0 {
 				t.LangID = _nb
 				changed = true
 			}
 			break
 		}
 	}
 	if c&DeprecatedScript != 0 {
 		if t.ScriptID == _Qaai {
 			changed = true
 			t.ScriptID = _Zinh
 		}
 	}
 	if c&DeprecatedRegion != 0 {
 		if r := t.RegionID.Canonicalize(); r != t.RegionID {
 			changed = true
 			t.RegionID = r
 		}
 	}
 	return t, changed
 }

 // Canonicalize returns the canonicalized equivalent of the tag.
 func (c CanonType) Canonicalize(t Tag) (Tag, error) {
 	// First try fast path.
 	if t.isCompact() {
 		if _, changed := canonicalize(c, compact.Tag(t).Tag()); !changed {
 			return t, nil
 		}
 	}
 	// It is unlikely that one will canonicalize a tag after matching. So do
 	// a slow but simple approach here.
 	if tag, changed := canonicalize(c, t.tag()); changed {
 		tag.RemakeString()
 		return makeTag(tag), nil
 	}
 	return t, nil

 }

 // Confidence indicates the level of certainty for a given return value.
 // For example, Serbian may be written in Cyrillic or Latin script.
 // The confidence level indicates whether a value was explicitly specified,
 // whether it is typically the only possible value, or whether there is
 // an ambiguity.
 type Confidence int

 const (
 	No    Confidence = iota // full confidence that there was no match
 	Low                     // most likely value picked out of a set of alternatives
 	High                    // value is generally assumed to be the correct match
 	Exact                   // exact match or explicitly specified value
 )

 var confName = []string{"No", "Low", "High", "Exact"}

 func (c Confidence) String() string {
 	return confName[c]
 }

 // String returns the canonical string representation of the language tag.
 func (t Tag) String() string {
 	return t.tag().String()
 }

 // MarshalText implements encoding.TextMarshaler.
 func (t Tag) MarshalText() (text []byte, err error) {
 	return t.tag().MarshalText()
 }

 // UnmarshalText implements encoding.TextUnmarshaler.
 func (t *Tag) UnmarshalText(text []byte) error {
 	var tag language.Tag
 	err := tag.UnmarshalText(text)
 	*t = makeTag(tag)
 	return err
 }

 // Base returns the base language of the language tag. If the base language is
 // unspecified, an attempt will be made to infer it from the context.
 // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
 func (t Tag) Base() (Base, Confidence) {
 	if b := t.lang(); b != 0 {
 		return Base{b}, Exact
 	}
 	tt := t.tag()
 	c := High
 	if tt.ScriptID == 0 && !tt.RegionID.IsCountry() {
 		c = Low
 	}
 	if tag, err := tt.Maximize(); err == nil && tag.LangID != 0 {
 		return Base{tag.LangID}, c
 	}
 	return Base{0}, No
 }

 // Script infers the script for the language tag. If it was not explicitly given, it will infer
 // a most likely candidate.
 // If more than one script is commonly used for a language, the most likely one
 // is returned with a low confidence indication. For example, it returns (Cyrl, Low)
 // for Serbian.
 // If a script cannot be inferred (Zzzz, No) is returned. We do not use Zyyy (undetermined)
 // as one would suspect from the IANA registry for BCP 47. In a Unicode context Zyyy marks
 // common characters (like 1, 2, 3, '.', etc.) and is therefore more like multiple scripts.
 // See https://www.unicode.org/reports/tr24/#Values for more details. Zzzz is also used for
 // unknown value in CLDR.  (Zzzz, Exact) is returned if Zzzz was explicitly specified.
 // Note that an inferred script is never guaranteed to be the correct one. Latin is
 // almost exclusively used for Afrikaans, but Arabic has been used for some texts
 // in the past.  Also, the script that is commonly used may change over time.
 // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
 func (t Tag) Script() (Script, Confidence) {
 	if scr := t.script(); scr != 0 {
 		return Script{scr}, Exact
 	}
 	tt := t.tag()
 	sc, c := language.Script(_Zzzz), No
 	if scr := tt.LangID.SuppressScript(); scr != 0 {
 		// Note: it is not always the case that a language with a suppress
 		// script value is only written in one script (e.g. kk, ms, pa).
 		if tt.RegionID == 0 {
 			return Script{scr}, High
 		}
 		sc, c = scr, High
 	}
 	if tag, err := tt.Maximize(); err == nil {
 		if tag.ScriptID != sc {
 			sc, c = tag.ScriptID, Low
 		}
 	} else {
 		tt, _ = canonicalize(Deprecated|Macro, tt)
 		if tag, err := tt.Maximize(); err == nil && tag.ScriptID != sc {
 			sc, c = tag.ScriptID, Low
 		}
 	}
 	return Script{sc}, c
 }

 // Region returns the region for the language tag. If it was not explicitly given, it will
 // infer a most likely candidate from the context.
 // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
 func (t Tag) Region() (Region, Confidence) {
 	if r := t.region(); r != 0 {
 		return Region{r}, Exact
 	}
 	tt := t.tag()
 	if tt, err := tt.Maximize(); err == nil {
 		return Region{tt.RegionID}, Low // TODO: differentiate between high and low.
 	}
 	tt, _ = canonicalize(Deprecated|Macro, tt)
 	if tag, err := tt.Maximize(); err == nil {
 		return Region{tag.RegionID}, Low
 	}
 	return Region{_ZZ}, No // TODO: return world instead of undetermined?
 }

 // Variants returns the variants specified explicitly for this language tag.
 // or nil if no variant was specified.
 func (t Tag) Variants() []Variant {
 	if !compact.Tag(t).MayHaveVariants() {
 		return nil
 	}
 	v := []Variant{}
 	x, str := "", t.tag().Variants()
 	for str != "" {
 		x, str = nextToken(str)
 		v = append(v, Variant{x})
 	}
 	return v
 }

 // Parent returns the CLDR parent of t. In CLDR, missing fields in data for a
 // specific language are substituted with fields from the parent language.
 // The parent for a language may change for newer versions of CLDR.
 //
 // Parent returns a tag for a less specific language that is mutually
 // intelligible or Und if there is no such language. This may not be the same as
 // simply stripping the last BCP 47 subtag. For instance, the parent of "zh-TW"
 // is "zh-Hant", and the parent of "zh-Hant" is "und".
 func (t Tag) Parent() Tag {
 	return Tag(compact.Tag(t).Parent())
 }

 // returns token t and the rest of the string.
 func nextToken(s string) (t, tail string) {
 	p := strings.Index(s[1:], "-")
 	if p == -1 {
 		return s[1:], ""
 	}
 	p++
 	return s[1:p], s[p:]
 }

 // Extension is a single BCP 47 extension.
 type Extension struct {
 	s string
 }

 // String returns the string representation of the extension, including the
 // type tag.
 func (e Extension) String() string {
 	return e.s
 }

 // ParseExtension parses s as an extension and returns it on success.
 func ParseExtension(s string) (e Extension, err error) {
 	ext, err := language.ParseExtension(s)
 	return Extension{ext}, err
 }

 // Type returns the one-byte extension type of e. It returns 0 for the zero
 // exception.
 func (e Extension) Type() byte {
 	if e.s == "" {
 		return 0
 	}
 	return e.s[0]
 }

 // Tokens returns the list of tokens of e.
 func (e Extension) Tokens() []string {
 	return strings.Split(e.s, "-")
 }

 // Extension returns the extension of type x for tag t. It will return
 // false for ok if t does not have the requested extension. The returned
 // extension will be invalid in this case.
 func (t Tag) Extension(x byte) (ext Extension, ok bool) {
 	if !compact.Tag(t).MayHaveExtensions() {
 		return Extension{}, false
 	}
 	e, ok := t.tag().Extension(x)
 	return Extension{e}, ok
 }

 // Extensions returns all extensions of t.
 func (t Tag) Extensions() []Extension {
 	if !compact.Tag(t).MayHaveExtensions() {
 		return nil
 	}
 	e := []Extension{}
 	for _, ext := range t.tag().Extensions() {
 		e = append(e, Extension{ext})
 	}
 	return e
 }

 // TypeForKey returns the type associated with the given key, where key and type
 // are of the allowed values defined for the Unicode locale extension ('u') in
 // https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
 // TypeForKey will traverse the inheritance chain to get the correct value.
 //
 // If there are multiple types associated with a key, only the first will be
 // returned. If there is no type associated with a key, it returns the empty
 // string.
 func (t Tag) TypeForKey(key string) string {
 	if !compact.Tag(t).MayHaveExtensions() {
 		if key != "rg" && key != "va" {
 			return ""
 		}
 	}
 	return t.tag().TypeForKey(key)
 }

 // SetTypeForKey returns a new Tag with the key set to type, where key and type
 // are of the allowed values defined for the Unicode locale extension ('u') in
 // https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
 // An empty value removes an existing pair with the same key.
 func (t Tag) SetTypeForKey(key, value string) (Tag, error) {
 	tt, err := t.tag().SetTypeForKey(key, value)
 	return makeTag(tt), err
 }

 // NumCompactTags is the number of compact tags. The maximum tag is
 // NumCompactTags-1.
 const NumCompactTags = compact.NumCompactTags

 // CompactIndex returns an index, where 0 <= index < NumCompactTags, for tags
 // for which data exists in the text repository.The index will change over time
 // and should not be stored in persistent storage. If t does not match a compact
 // index, exact will be false and the compact index will be returned for the
 // first match after repeatedly taking the Parent of t.
 func CompactIndex(t Tag) (index int, exact bool) {
 	id, exact := compact.LanguageID(compact.Tag(t))
 	return int(id), exact
 }

 var root = language.Tag{}

 // Base is an ISO 639 language code, used for encoding the base language
 // of a language tag.
 type Base struct {
 	langID language.Language
 }

 // ParseBase parses a 2- or 3-letter ISO 639 code.
 // It returns a ValueError if s is a well-formed but unknown language identifier
 // or another error if another error occurred.
 func ParseBase(s string) (Base, error) {
 	l, err := language.ParseBase(s)
 	return Base{l}, err
 }

 // String returns the BCP 47 representation of the base language.
 func (b Base) String() string {
 	return b.langID.String()
 }

 // ISO3 returns the ISO 639-3 language code.
 func (b Base) ISO3() string {
 	return b.langID.ISO3()
 }

 // IsPrivateUse reports whether this language code is reserved for private use.
 func (b Base) IsPrivateUse() bool {
 	return b.langID.IsPrivateUse()
 }

 // Script is a 4-letter ISO 15924 code for representing scripts.
 // It is idiomatically represented in title case.
 type Script struct {
 	scriptID language.Script
 }

 // ParseScript parses a 4-letter ISO 15924 code.
 // It returns a ValueError if s is a well-formed but unknown script identifier
 // or another error if another error occurred.
 func ParseScript(s string) (Script, error) {
 	sc, err := language.ParseScript(s)
 	return Script{sc}, err
 }

 // String returns the script code in title case.
 // It returns "Zzzz" for an unspecified script.
 func (s Script) String() string {
 	return s.scriptID.String()
 }

 // IsPrivateUse reports whether this script code is reserved for private use.
 func (s Script) IsPrivateUse() bool {
 	return s.scriptID.IsPrivateUse()
 }

 // Region is an ISO 3166-1 or UN M.49 code for representing countries and regions.
 type Region struct {
 	regionID language.Region
 }

 // EncodeM49 returns the Region for the given UN M.49 code.
 // It returns an error if r is not a valid code.
 func EncodeM49(r int) (Region, error) {
 	rid, err := language.EncodeM49(r)
 	return Region{rid}, err
 }

 // ParseRegion parses a 2- or 3-letter ISO 3166-1 or a UN M.49 code.
 // It returns a ValueError if s is a well-formed but unknown region identifier
 // or another error if another error occurred.
 func ParseRegion(s string) (Region, error) {
 	r, err := language.ParseRegion(s)
 	return Region{r}, err
 }

 // String returns the BCP 47 representation for the region.
 // It returns "ZZ" for an unspecified region.
 func (r Region) String() string {
 	return r.regionID.String()
 }

 // ISO3 returns the 3-letter ISO code of r.
 // Note that not all regions have a 3-letter ISO code.
 // In such cases this method returns "ZZZ".
 func (r Region) ISO3() string {
 	return r.regionID.ISO3()
 }

 // M49 returns the UN M.49 encoding of r, or 0 if this encoding
 // is not defined for r.
 func (r Region) M49() int {
 	return r.regionID.M49()
 }

 // IsPrivateUse reports whether r has the ISO 3166 User-assigned status. This
 // may include private-use tags that are assigned by CLDR and used in this
 // implementation. So IsPrivateUse and IsCountry can be simultaneously true.
 func (r Region) IsPrivateUse() bool {
 	return r.regionID.IsPrivateUse()
 }

 // IsCountry returns whether this region is a country or autonomous area. This
 // includes non-standard definitions from CLDR.
 func (r Region) IsCountry() bool {
 	return r.regionID.IsCountry()
 }

 // IsGroup returns whether this region defines a collection of regions. This
 // includes non-standard definitions from CLDR.
 func (r Region) IsGroup() bool {
 	return r.regionID.IsGroup()
 }

 // Contains returns whether Region c is contained by Region r. It returns true
 // if c == r.
 func (r Region) Contains(c Region) bool {
 	return r.regionID.Contains(c.regionID)
 }

 // TLD returns the country code top-level domain (ccTLD). UK is returned for GB.
 // In all other cases it returns either the region itself or an error.
 //
 // This method may return an error for a region for which there exists a
 // canonical form with a ccTLD. To get that ccTLD canonicalize r first. The
 // region will already be canonicalized it was obtained from a Tag that was
 // obtained using any of the default methods.
 func (r Region) TLD() (Region, error) {
 	tld, err := r.regionID.TLD()
 	return Region{tld}, err
 }

 // Canonicalize returns the region or a possible replacement if the region is
 // deprecated. It will not return a replacement for deprecated regions that
 // are split into multiple regions.
 func (r Region) Canonicalize() Region {
 	return Region{r.regionID.Canonicalize()}
 }

 // Variant represents a registered variant of a language as defined by BCP 47.
 type Variant struct {
 	variant string
 }

 // ParseVariant parses and returns a Variant. An error is returned if s is not
 // a valid variant.
 func ParseVariant(s string) (Variant, error) {
 	v, err := language.ParseVariant(s)
 	return Variant{v.String()}, err
 }

 // String returns the string representation of the variant.
 func (v Variant) String() string {
 	return v.variant
 }
	// Copyright 2013 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	//go:generate go run gen.go -output tables.go

	package language

	// TODO: Remove above NOTE after:
	// - verifying that tables are dropped correctly (most notably matcher tables).

	import (
	"strings"

	"golang.org/x/text/internal/language"
	"golang.org/x/text/internal/language/compact"
	)

	// Tag represents a BCP 47 language tag. It is used to specify an instance of a
	// specific language or locale. All language tag values are guaranteed to be
	// well-formed.
	type Tag compact.Tag

	func makeTag(t language.Tag) (tag Tag) {
	return Tag(compact.Make(t))
	}

	func (t *Tag) tag() language.Tag {
	return (*compact.Tag)(t).Tag()
	}

	func (t *Tag) isCompact() bool {
	return (*compact.Tag)(t).IsCompact()
	}

	// TODO: improve performance.
	func (t *Tag) lang() language.Language { return t.tag().LangID }
	func (t *Tag) region() language.Region { return t.tag().RegionID }
	func (t *Tag) script() language.Script { return t.tag().ScriptID }

	// Make is a convenience wrapper for Parse that omits the error.
	// In case of an error, a sensible default is returned.
	func Make(s string) Tag {
	return Default.Make(s)
	}

	// Make is a convenience wrapper for c.Parse that omits the error.
	// In case of an error, a sensible default is returned.
	func (c CanonType) Make(s string) Tag {
	t, _ := c.Parse(s)
	return t
	}

	// Raw returns the raw base language, script and region, without making an
	// attempt to infer their values.
	func (t Tag) Raw() (b Base, s Script, r Region) {
	tt := t.tag()
	return Base{tt.LangID}, Script{tt.ScriptID}, Region{tt.RegionID}
	}

	// IsRoot returns true if t is equal to language "und".
	func (t Tag) IsRoot() bool {
	return compact.Tag(t).IsRoot()
	}

	// CanonType can be used to enable or disable various types of canonicalization.
	type CanonType int

	const (
	// Replace deprecated base languages with their preferred replacements.
	DeprecatedBase CanonType = 1 << iota
	// Replace deprecated scripts with their preferred replacements.
	DeprecatedScript
	// Replace deprecated regions with their preferred replacements.
	DeprecatedRegion
	// Remove redundant scripts.
	SuppressScript
	// Normalize legacy encodings. This includes legacy languages defined in
	// CLDR as well as bibliographic codes defined in ISO-639.
	Legacy
	// Map the dominant language of a macro language group to the macro language
	// subtag. For example cmn -> zh.
	Macro
	// The CLDR flag should be used if full compatibility with CLDR is required.
	// There are a few cases where language.Tag may differ from CLDR. To follow all
	// of CLDR's suggestions, use All\|CLDR.
	CLDR

	// Raw can be used to Compose or Parse without Canonicalization.
	Raw CanonType = 0

	// Replace all deprecated tags with their preferred replacements.
	Deprecated = DeprecatedBase \| DeprecatedScript \| DeprecatedRegion

	// All canonicalizations recommended by BCP 47.
	BCP47 = Deprecated \| SuppressScript

	// All canonicalizations.
	All = BCP47 \| Legacy \| Macro

	// Default is the canonicalization used by Parse, Make and Compose. To
	// preserve as much information as possible, canonicalizations that remove
	// potentially valuable information are not included. The Matcher is
	// designed to recognize similar tags that would be the same if
	// they were canonicalized using All.
	Default = Deprecated \| Legacy

	canonLang = DeprecatedBase \| Legacy \| Macro

	// TODO: LikelyScript, LikelyRegion: suppress similar to ICU.
	)

	// canonicalize returns the canonicalized equivalent of the tag and
	// whether there was any change.
	func canonicalize(c CanonType, t language.Tag) (language.Tag, bool) {
	if c == Raw {
	return t, false
	}
	changed := false
	if c&SuppressScript != 0 {
	if t.LangID.SuppressScript() == t.ScriptID {
	t.ScriptID = 0
	changed = true
	}
	}
	if c&canonLang != 0 {
	for {
	if l, aliasType := t.LangID.Canonicalize(); l != t.LangID {
	switch aliasType {
	case language.Legacy:
	if c&Legacy != 0 {
	if t.LangID == _sh && t.ScriptID == 0 {
	t.ScriptID = _Latn
	}
	t.LangID = l
	changed = true
	}
	case language.Macro:
	if c&Macro != 0 {
	// We deviate here from CLDR. The mapping "nb" -> "no"
	// qualifies as a typical Macro language mapping. However,
	// for legacy reasons, CLDR maps "no", the macro language
	// code for Norwegian, to the dominant variant "nb". This
	// change is currently under consideration for CLDR as well.
	// See https://unicode.org/cldr/trac/ticket/2698 and also
	// https://unicode.org/cldr/trac/ticket/1790 for some of the
	// practical implications. TODO: this check could be removed
	// if CLDR adopts this change.
	if c&CLDR == 0 \|\| t.LangID != _nb {
	changed = true
	t.LangID = l
	}
	}
	case language.Deprecated:
	if c&DeprecatedBase != 0 {
	if t.LangID == _mo && t.RegionID == 0 {
	t.RegionID = _MD
	}
	t.LangID = l
	changed = true
	// Other canonicalization types may still apply.
	continue
	}
	}
	} else if c&Legacy != 0 && t.LangID == _no && c&CLDR != 0 {
	t.LangID = _nb
	changed = true
	}
	break
	}
	}
	if c&DeprecatedScript != 0 {
	if t.ScriptID == _Qaai {
	changed = true
	t.ScriptID = _Zinh
	}
	}
	if c&DeprecatedRegion != 0 {
	if r := t.RegionID.Canonicalize(); r != t.RegionID {
	changed = true
	t.RegionID = r
	}
	}
	return t, changed
	}

	// Canonicalize returns the canonicalized equivalent of the tag.
	func (c CanonType) Canonicalize(t Tag) (Tag, error) {
	// First try fast path.
	if t.isCompact() {
	if _, changed := canonicalize(c, compact.Tag(t).Tag()); !changed {
	return t, nil
	}
	}
	// It is unlikely that one will canonicalize a tag after matching. So do
	// a slow but simple approach here.
	if tag, changed := canonicalize(c, t.tag()); changed {
	tag.RemakeString()
	return makeTag(tag), nil
	}
	return t, nil

	}

	// Confidence indicates the level of certainty for a given return value.
	// For example, Serbian may be written in Cyrillic or Latin script.
	// The confidence level indicates whether a value was explicitly specified,
	// whether it is typically the only possible value, or whether there is
	// an ambiguity.
	type Confidence int

	const (
	No Confidence = iota // full confidence that there was no match
	Low // most likely value picked out of a set of alternatives
	High // value is generally assumed to be the correct match
	Exact // exact match or explicitly specified value
	)

	var confName = []string{"No", "Low", "High", "Exact"}

	func (c Confidence) String() string {
	return confName[c]
	}

	// String returns the canonical string representation of the language tag.
	func (t Tag) String() string {
	return t.tag().String()
	}

	// MarshalText implements encoding.TextMarshaler.
	func (t Tag) MarshalText() (text []byte, err error) {
	return t.tag().MarshalText()
	}

	// UnmarshalText implements encoding.TextUnmarshaler.
	func (t *Tag) UnmarshalText(text []byte) error {
	var tag language.Tag
	err := tag.UnmarshalText(text)
	*t = makeTag(tag)
	return err
	}

	// Base returns the base language of the language tag. If the base language is
	// unspecified, an attempt will be made to infer it from the context.
	// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
	func (t Tag) Base() (Base, Confidence) {
	if b := t.lang(); b != 0 {
	return Base{b}, Exact
	}
	tt := t.tag()
	c := High
	if tt.ScriptID == 0 && !tt.RegionID.IsCountry() {
	c = Low
	}
	if tag, err := tt.Maximize(); err == nil && tag.LangID != 0 {
	return Base{tag.LangID}, c
	}
	return Base{0}, No
	}

	// Script infers the script for the language tag. If it was not explicitly given, it will infer
	// a most likely candidate.
	// If more than one script is commonly used for a language, the most likely one
	// is returned with a low confidence indication. For example, it returns (Cyrl, Low)
	// for Serbian.
	// If a script cannot be inferred (Zzzz, No) is returned. We do not use Zyyy (undetermined)
	// as one would suspect from the IANA registry for BCP 47. In a Unicode context Zyyy marks
	// common characters (like 1, 2, 3, '.', etc.) and is therefore more like multiple scripts.
	// See https://www.unicode.org/reports/tr24/#Values for more details. Zzzz is also used for
	// unknown value in CLDR. (Zzzz, Exact) is returned if Zzzz was explicitly specified.
	// Note that an inferred script is never guaranteed to be the correct one. Latin is
	// almost exclusively used for Afrikaans, but Arabic has been used for some texts
	// in the past. Also, the script that is commonly used may change over time.
	// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
	func (t Tag) Script() (Script, Confidence) {
	if scr := t.script(); scr != 0 {
	return Script{scr}, Exact
	}
	tt := t.tag()
	sc, c := language.Script(_Zzzz), No
	if scr := tt.LangID.SuppressScript(); scr != 0 {
	// Note: it is not always the case that a language with a suppress
	// script value is only written in one script (e.g. kk, ms, pa).
	if tt.RegionID == 0 {
	return Script{scr}, High
	}
	sc, c = scr, High
	}
	if tag, err := tt.Maximize(); err == nil {
	if tag.ScriptID != sc {
	sc, c = tag.ScriptID, Low
	}
	} else {
	tt, _ = canonicalize(Deprecated\|Macro, tt)
	if tag, err := tt.Maximize(); err == nil && tag.ScriptID != sc {
	sc, c = tag.ScriptID, Low
	}
	}
	return Script{sc}, c
	}

	// Region returns the region for the language tag. If it was not explicitly given, it will
	// infer a most likely candidate from the context.
	// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
	func (t Tag) Region() (Region, Confidence) {
	if r := t.region(); r != 0 {
	return Region{r}, Exact
	}
	tt := t.tag()
	if tt, err := tt.Maximize(); err == nil {
	return Region{tt.RegionID}, Low // TODO: differentiate between high and low.
	}
	tt, _ = canonicalize(Deprecated\|Macro, tt)
	if tag, err := tt.Maximize(); err == nil {
	return Region{tag.RegionID}, Low
	}
	return Region{_ZZ}, No // TODO: return world instead of undetermined?
	}

	// Variants returns the variants specified explicitly for this language tag.
	// or nil if no variant was specified.
	func (t Tag) Variants() []Variant {
	if !compact.Tag(t).MayHaveVariants() {
	return nil
	}
	v := []Variant{}
	x, str := "", t.tag().Variants()
	for str != "" {
	x, str = nextToken(str)
	v = append(v, Variant{x})
	}
	return v
	}

	// Parent returns the CLDR parent of t. In CLDR, missing fields in data for a
	// specific language are substituted with fields from the parent language.
	// The parent for a language may change for newer versions of CLDR.
	//
	// Parent returns a tag for a less specific language that is mutually
	// intelligible or Und if there is no such language. This may not be the same as
	// simply stripping the last BCP 47 subtag. For instance, the parent of "zh-TW"
	// is "zh-Hant", and the parent of "zh-Hant" is "und".
	func (t Tag) Parent() Tag {
	return Tag(compact.Tag(t).Parent())
	}

	// returns token t and the rest of the string.
	func nextToken(s string) (t, tail string) {
	p := strings.Index(s[1:], "-")
	if p == -1 {
	return s[1:], ""
	}
	p++
	return s[1:p], s[p:]
	}

	// Extension is a single BCP 47 extension.
	type Extension struct {
	s string
	}

	// String returns the string representation of the extension, including the
	// type tag.
	func (e Extension) String() string {
	return e.s
	}

	// ParseExtension parses s as an extension and returns it on success.
	func ParseExtension(s string) (e Extension, err error) {
	ext, err := language.ParseExtension(s)
	return Extension{ext}, err
	}

	// Type returns the one-byte extension type of e. It returns 0 for the zero
	// exception.
	func (e Extension) Type() byte {
	if e.s == "" {
	return 0
	}
	return e.s[0]
	}

	// Tokens returns the list of tokens of e.
	func (e Extension) Tokens() []string {
	return strings.Split(e.s, "-")
	}

	// Extension returns the extension of type x for tag t. It will return
	// false for ok if t does not have the requested extension. The returned
	// extension will be invalid in this case.
	func (t Tag) Extension(x byte) (ext Extension, ok bool) {
	if !compact.Tag(t).MayHaveExtensions() {
	return Extension{}, false
	}
	e, ok := t.tag().Extension(x)
	return Extension{e}, ok
	}

	// Extensions returns all extensions of t.
	func (t Tag) Extensions() []Extension {
	if !compact.Tag(t).MayHaveExtensions() {
	return nil
	}
	e := []Extension{}
	for _, ext := range t.tag().Extensions() {
	e = append(e, Extension{ext})
	}
	return e
	}

	// TypeForKey returns the type associated with the given key, where key and type
	// are of the allowed values defined for the Unicode locale extension ('u') in
	// https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
	// TypeForKey will traverse the inheritance chain to get the correct value.
	//
	// If there are multiple types associated with a key, only the first will be
	// returned. If there is no type associated with a key, it returns the empty
	// string.
	func (t Tag) TypeForKey(key string) string {
	if !compact.Tag(t).MayHaveExtensions() {
	if key != "rg" && key != "va" {
	return ""
	}
	}
	return t.tag().TypeForKey(key)
	}

	// SetTypeForKey returns a new Tag with the key set to type, where key and type
	// are of the allowed values defined for the Unicode locale extension ('u') in
	// https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
	// An empty value removes an existing pair with the same key.
	func (t Tag) SetTypeForKey(key, value string) (Tag, error) {
	tt, err := t.tag().SetTypeForKey(key, value)
	return makeTag(tt), err
	}

	// NumCompactTags is the number of compact tags. The maximum tag is
	// NumCompactTags-1.
	const NumCompactTags = compact.NumCompactTags

	// CompactIndex returns an index, where 0 <= index < NumCompactTags, for tags
	// for which data exists in the text repository.The index will change over time
	// and should not be stored in persistent storage. If t does not match a compact
	// index, exact will be false and the compact index will be returned for the
	// first match after repeatedly taking the Parent of t.
	func CompactIndex(t Tag) (index int, exact bool) {
	id, exact := compact.LanguageID(compact.Tag(t))
	return int(id), exact
	}

	var root = language.Tag{}

	// Base is an ISO 639 language code, used for encoding the base language
	// of a language tag.
	type Base struct {
	langID language.Language
	}

	// ParseBase parses a 2- or 3-letter ISO 639 code.
	// It returns a ValueError if s is a well-formed but unknown language identifier
	// or another error if another error occurred.
	func ParseBase(s string) (Base, error) {
	l, err := language.ParseBase(s)
	return Base{l}, err
	}

	// String returns the BCP 47 representation of the base language.
	func (b Base) String() string {
	return b.langID.String()
	}

	// ISO3 returns the ISO 639-3 language code.
	func (b Base) ISO3() string {
	return b.langID.ISO3()
	}

	// IsPrivateUse reports whether this language code is reserved for private use.
	func (b Base) IsPrivateUse() bool {
	return b.langID.IsPrivateUse()
	}

	// Script is a 4-letter ISO 15924 code for representing scripts.
	// It is idiomatically represented in title case.
	type Script struct {
	scriptID language.Script
	}

	// ParseScript parses a 4-letter ISO 15924 code.
	// It returns a ValueError if s is a well-formed but unknown script identifier
	// or another error if another error occurred.
	func ParseScript(s string) (Script, error) {
	sc, err := language.ParseScript(s)
	return Script{sc}, err
	}

	// String returns the script code in title case.
	// It returns "Zzzz" for an unspecified script.
	func (s Script) String() string {
	return s.scriptID.String()
	}

	// IsPrivateUse reports whether this script code is reserved for private use.
	func (s Script) IsPrivateUse() bool {
	return s.scriptID.IsPrivateUse()
	}

	// Region is an ISO 3166-1 or UN M.49 code for representing countries and regions.
	type Region struct {
	regionID language.Region
	}

	// EncodeM49 returns the Region for the given UN M.49 code.
	// It returns an error if r is not a valid code.
	func EncodeM49(r int) (Region, error) {
	rid, err := language.EncodeM49(r)
	return Region{rid}, err
	}

	// ParseRegion parses a 2- or 3-letter ISO 3166-1 or a UN M.49 code.
	// It returns a ValueError if s is a well-formed but unknown region identifier
	// or another error if another error occurred.
	func ParseRegion(s string) (Region, error) {
	r, err := language.ParseRegion(s)
	return Region{r}, err
	}

	// String returns the BCP 47 representation for the region.
	// It returns "ZZ" for an unspecified region.
	func (r Region) String() string {
	return r.regionID.String()
	}

	// ISO3 returns the 3-letter ISO code of r.
	// Note that not all regions have a 3-letter ISO code.
	// In such cases this method returns "ZZZ".
	func (r Region) ISO3() string {
	return r.regionID.ISO3()
	}

	// M49 returns the UN M.49 encoding of r, or 0 if this encoding
	// is not defined for r.
	func (r Region) M49() int {
	return r.regionID.M49()
	}

	// IsPrivateUse reports whether r has the ISO 3166 User-assigned status. This
	// may include private-use tags that are assigned by CLDR and used in this
	// implementation. So IsPrivateUse and IsCountry can be simultaneously true.
	func (r Region) IsPrivateUse() bool {
	return r.regionID.IsPrivateUse()
	}

	// IsCountry returns whether this region is a country or autonomous area. This
	// includes non-standard definitions from CLDR.
	func (r Region) IsCountry() bool {
	return r.regionID.IsCountry()
	}

	// IsGroup returns whether this region defines a collection of regions. This
	// includes non-standard definitions from CLDR.
	func (r Region) IsGroup() bool {
	return r.regionID.IsGroup()
	}

	// Contains returns whether Region c is contained by Region r. It returns true
	// if c == r.
	func (r Region) Contains(c Region) bool {
	return r.regionID.Contains(c.regionID)
	}

	// TLD returns the country code top-level domain (ccTLD). UK is returned for GB.
	// In all other cases it returns either the region itself or an error.
	//
	// This method may return an error for a region for which there exists a
	// canonical form with a ccTLD. To get that ccTLD canonicalize r first. The
	// region will already be canonicalized it was obtained from a Tag that was
	// obtained using any of the default methods.
	func (r Region) TLD() (Region, error) {
	tld, err := r.regionID.TLD()
	return Region{tld}, err
	}

	// Canonicalize returns the region or a possible replacement if the region is
	// deprecated. It will not return a replacement for deprecated regions that
	// are split into multiple regions.
	func (r Region) Canonicalize() Region {
	return Region{r.regionID.Canonicalize()}
	}

	// Variant represents a registered variant of a language as defined by BCP 47.
	type Variant struct {
	variant string
	}

	// ParseVariant parses and returns a Variant. An error is returned if s is not
	// a valid variant.
	func ParseVariant(s string) (Variant, error) {
	v, err := language.ParseVariant(s)
	return Variant{v.String()}, err
	}

	// String returns the string representation of the variant.
	func (v Variant) String() string {
	return v.variant
	}