internal/language/language.go - text - Git at Google

 // Copyright 2013 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 //go:generate go run gen.go gen_common.go -output tables.go

 package language // import "golang.org/x/text/internal/language"

 // TODO: Remove above NOTE after:
 // - verifying that tables are dropped correctly (most notably matcher tables).

 import (
 	"errors"
 	"fmt"
 	"strings"
 )

 const (
 	// maxCoreSize is the maximum size of a BCP 47 tag without variants and
 	// extensions. Equals max lang (3) + script (4) + max reg (3) + 2 dashes.
 	maxCoreSize = 12

 	// max99thPercentileSize is a somewhat arbitrary buffer size that presumably
 	// is large enough to hold at least 99% of the BCP 47 tags.
 	max99thPercentileSize = 32

 	// maxSimpleUExtensionSize is the maximum size of a -u extension with one
 	// key-type pair. Equals len("-u-") + key (2) + dash + max value (8).
 	maxSimpleUExtensionSize = 14
 )

 // Tag represents a BCP 47 language tag. It is used to specify an instance of a
 // specific language or locale. All language tag values are guaranteed to be
 // well-formed. The zero value of Tag is Und.
 type Tag struct {
 	// TODO: the following fields have the form TagTypeID. This name is chosen
 	// to allow refactoring the public package without conflicting with its
 	// Base, Script, and Region methods. Once the transition is fully completed
 	// the ID can be stripped from the name.

 	LangID   Language
 	RegionID Region
 	// TODO: we will soon run out of positions for ScriptID. Idea: instead of
 	// storing lang, region, and ScriptID codes, store only the compact index and
 	// have a lookup table from this code to its expansion. This greatly speeds
 	// up table lookup, speed up common variant cases.
 	// This will also immediately free up 3 extra bytes. Also, the pVariant
 	// field can now be moved to the lookup table, as the compact index uniquely
 	// determines the offset of a possible variant.
 	ScriptID Script
 	pVariant byte   // offset in str, includes preceding '-'
 	pExt     uint16 // offset of first extension, includes preceding '-'

 	// str is the string representation of the Tag. It will only be used if the
 	// tag has variants or extensions.
 	str string
 }

 // Make is a convenience wrapper for Parse that omits the error.
 // In case of an error, a sensible default is returned.
 func Make(s string) Tag {
 	t, _ := Parse(s)
 	return t
 }

 // Raw returns the raw base language, script and region, without making an
 // attempt to infer their values.
 // TODO: consider removing
 func (t Tag) Raw() (b Language, s Script, r Region) {
 	return t.LangID, t.ScriptID, t.RegionID
 }

 // equalTags compares language, script and region subtags only.
 func (t Tag) equalTags(a Tag) bool {
 	return t.LangID == a.LangID && t.ScriptID == a.ScriptID && t.RegionID == a.RegionID
 }

 // IsRoot returns true if t is equal to language "und".
 func (t Tag) IsRoot() bool {
 	if int(t.pVariant) < len(t.str) {
 		return false
 	}
 	return t.equalTags(Und)
 }

 // IsPrivateUse reports whether the Tag consists solely of an IsPrivateUse use
 // tag.
 func (t Tag) IsPrivateUse() bool {
 	return t.str != "" && t.pVariant == 0
 }

 // RemakeString is used to update t.str in case lang, script or region changed.
 // It is assumed that pExt and pVariant still point to the start of the
 // respective parts.
 func (t *Tag) RemakeString() {
 	if t.str == "" {
 		return
 	}
 	extra := t.str[t.pVariant:]
 	if t.pVariant > 0 {
 		extra = extra[1:]
 	}
 	if t.equalTags(Und) && strings.HasPrefix(extra, "x-") {
 		t.str = extra
 		t.pVariant = 0
 		t.pExt = 0
 		return
 	}
 	var buf [max99thPercentileSize]byte // avoid extra memory allocation in most cases.
 	b := buf[:t.genCoreBytes(buf[:])]
 	if extra != "" {
 		diff := len(b) - int(t.pVariant)
 		b = append(b, '-')
 		b = append(b, extra...)
 		t.pVariant = uint8(int(t.pVariant) + diff)
 		t.pExt = uint16(int(t.pExt) + diff)
 	} else {
 		t.pVariant = uint8(len(b))
 		t.pExt = uint16(len(b))
 	}
 	t.str = string(b)
 }

 // genCoreBytes writes a string for the base languages, script and region tags
 // to the given buffer and returns the number of bytes written. It will never
 // write more than maxCoreSize bytes.
 func (t *Tag) genCoreBytes(buf []byte) int {
 	n := t.LangID.StringToBuf(buf[:])
 	if t.ScriptID != 0 {
 		n += copy(buf[n:], "-")
 		n += copy(buf[n:], t.ScriptID.String())
 	}
 	if t.RegionID != 0 {
 		n += copy(buf[n:], "-")
 		n += copy(buf[n:], t.RegionID.String())
 	}
 	return n
 }

 // String returns the canonical string representation of the language tag.
 func (t Tag) String() string {
 	if t.str != "" {
 		return t.str
 	}
 	if t.ScriptID == 0 && t.RegionID == 0 {
 		return t.LangID.String()
 	}
 	buf := [maxCoreSize]byte{}
 	return string(buf[:t.genCoreBytes(buf[:])])
 }

 // MarshalText implements encoding.TextMarshaler.
 func (t Tag) MarshalText() (text []byte, err error) {
 	if t.str != "" {
 		text = append(text, t.str...)
 	} else if t.ScriptID == 0 && t.RegionID == 0 {
 		text = append(text, t.LangID.String()...)
 	} else {
 		buf := [maxCoreSize]byte{}
 		text = buf[:t.genCoreBytes(buf[:])]
 	}
 	return text, nil
 }

 // UnmarshalText implements encoding.TextUnmarshaler.
 func (t *Tag) UnmarshalText(text []byte) error {
 	tag, err := Parse(string(text))
 	*t = tag
 	return err
 }

 // Variants returns the part of the tag holding all variants or the empty string
 // if there are no variants defined.
 func (t Tag) Variants() string {
 	if t.pVariant == 0 {
 		return ""
 	}
 	return t.str[t.pVariant:t.pExt]
 }

 // VariantOrPrivateUseTags returns variants or private use tags.
 func (t Tag) VariantOrPrivateUseTags() string {
 	if t.pExt > 0 {
 		return t.str[t.pVariant:t.pExt]
 	}
 	return t.str[t.pVariant:]
 }

 // HasString reports whether this tag defines more than just the raw
 // components.
 func (t Tag) HasString() bool {
 	return t.str != ""
 }

 // Parent returns the CLDR parent of t. In CLDR, missing fields in data for a
 // specific language are substituted with fields from the parent language.
 // The parent for a language may change for newer versions of CLDR.
 func (t Tag) Parent() Tag {
 	if t.str != "" {
 		// Strip the variants and extensions.
 		b, s, r := t.Raw()
 		t = Tag{LangID: b, ScriptID: s, RegionID: r}
 		if t.RegionID == 0 && t.ScriptID != 0 && t.LangID != 0 {
 			base, _ := addTags(Tag{LangID: t.LangID})
 			if base.ScriptID == t.ScriptID {
 				return Tag{LangID: t.LangID}
 			}
 		}
 		return t
 	}
 	if t.LangID != 0 {
 		if t.RegionID != 0 {
 			maxScript := t.ScriptID
 			if maxScript == 0 {
 				max, _ := addTags(t)
 				maxScript = max.ScriptID
 			}

 			for i := range parents {
 				if Language(parents[i].lang) == t.LangID && Script(parents[i].maxScript) == maxScript {
 					for _, r := range parents[i].fromRegion {
 						if Region(r) == t.RegionID {
 							return Tag{
 								LangID:   t.LangID,
 								ScriptID: Script(parents[i].script),
 								RegionID: Region(parents[i].toRegion),
 							}
 						}
 					}
 				}
 			}

 			// Strip the script if it is the default one.
 			base, _ := addTags(Tag{LangID: t.LangID})
 			if base.ScriptID != maxScript {
 				return Tag{LangID: t.LangID, ScriptID: maxScript}
 			}
 			return Tag{LangID: t.LangID}
 		} else if t.ScriptID != 0 {
 			// The parent for an base-script pair with a non-default script is
 			// "und" instead of the base language.
 			base, _ := addTags(Tag{LangID: t.LangID})
 			if base.ScriptID != t.ScriptID {
 				return Und
 			}
 			return Tag{LangID: t.LangID}
 		}
 	}
 	return Und
 }

 // ParseExtension parses s as an extension and returns it on success.
 func ParseExtension(s string) (ext string, err error) {
 	defer func() {
 		if recover() != nil {
 			ext = ""
 			err = ErrSyntax
 		}
 	}()

 	scan := makeScannerString(s)
 	var end int
 	if n := len(scan.token); n != 1 {
 		return "", ErrSyntax
 	}
 	scan.toLower(0, len(scan.b))
 	end = parseExtension(&scan)
 	if end != len(s) {
 		return "", ErrSyntax
 	}
 	return string(scan.b), nil
 }

 // HasVariants reports whether t has variants.
 func (t Tag) HasVariants() bool {
 	return uint16(t.pVariant) < t.pExt
 }

 // HasExtensions reports whether t has extensions.
 func (t Tag) HasExtensions() bool {
 	return int(t.pExt) < len(t.str)
 }

 // Extension returns the extension of type x for tag t. It will return
 // false for ok if t does not have the requested extension. The returned
 // extension will be invalid in this case.
 func (t Tag) Extension(x byte) (ext string, ok bool) {
 	for i := int(t.pExt); i < len(t.str)-1; {
 		var ext string
 		i, ext = getExtension(t.str, i)
 		if ext[0] == x {
 			return ext, true
 		}
 	}
 	return "", false
 }

 // Extensions returns all extensions of t.
 func (t Tag) Extensions() []string {
 	e := []string{}
 	for i := int(t.pExt); i < len(t.str)-1; {
 		var ext string
 		i, ext = getExtension(t.str, i)
 		e = append(e, ext)
 	}
 	return e
 }

 // TypeForKey returns the type associated with the given key, where key and type
 // are of the allowed values defined for the Unicode locale extension ('u') in
 // https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
 // TypeForKey will traverse the inheritance chain to get the correct value.
 //
 // If there are multiple types associated with a key, only the first will be
 // returned. If there is no type associated with a key, it returns the empty
 // string.
 func (t Tag) TypeForKey(key string) string {
 	if _, start, end, _ := t.findTypeForKey(key); end != start {
 		s := t.str[start:end]
 		if p := strings.IndexByte(s, '-'); p >= 0 {
 			s = s[:p]
 		}
 		return s
 	}
 	return ""
 }

 var (
 	errPrivateUse       = errors.New("cannot set a key on a private use tag")
 	errInvalidArguments = errors.New("invalid key or type")
 )

 // SetTypeForKey returns a new Tag with the key set to type, where key and type
 // are of the allowed values defined for the Unicode locale extension ('u') in
 // https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
 // An empty value removes an existing pair with the same key.
 func (t Tag) SetTypeForKey(key, value string) (Tag, error) {
 	if t.IsPrivateUse() {
 		return t, errPrivateUse
 	}
 	if len(key) != 2 {
 		return t, errInvalidArguments
 	}

 	// Remove the setting if value is "".
 	if value == "" {
 		start, sep, end, _ := t.findTypeForKey(key)
 		if start != sep {
 			// Remove a possible empty extension.
 			switch {
 			case t.str[start-2] != '-': // has previous elements.
 			case end == len(t.str), // end of string
 				end+2 < len(t.str) && t.str[end+2] == '-': // end of extension
 				start -= 2
 			}
 			if start == int(t.pVariant) && end == len(t.str) {
 				t.str = ""
 				t.pVariant, t.pExt = 0, 0
 			} else {
 				t.str = fmt.Sprintf("%s%s", t.str[:start], t.str[end:])
 			}
 		}
 		return t, nil
 	}

 	if len(value) < 3 || len(value) > 8 {
 		return t, errInvalidArguments
 	}

 	var (
 		buf    [maxCoreSize + maxSimpleUExtensionSize]byte
 		uStart int // start of the -u extension.
 	)

 	// Generate the tag string if needed.
 	if t.str == "" {
 		uStart = t.genCoreBytes(buf[:])
 		buf[uStart] = '-'
 		uStart++
 	}

 	// Create new key-type pair and parse it to verify.
 	b := buf[uStart:]
 	copy(b, "u-")
 	copy(b[2:], key)
 	b[4] = '-'
 	b = b[:5+copy(b[5:], value)]
 	scan := makeScanner(b)
 	if parseExtensions(&scan); scan.err != nil {
 		return t, scan.err
 	}

 	// Assemble the replacement string.
 	if t.str == "" {
 		t.pVariant, t.pExt = byte(uStart-1), uint16(uStart-1)
 		t.str = string(buf[:uStart+len(b)])
 	} else {
 		s := t.str
 		start, sep, end, hasExt := t.findTypeForKey(key)
 		if start == sep {
 			if hasExt {
 				b = b[2:]
 			}
 			t.str = fmt.Sprintf("%s-%s%s", s[:sep], b, s[end:])
 		} else {
 			t.str = fmt.Sprintf("%s-%s%s", s[:start+3], value, s[end:])
 		}
 	}
 	return t, nil
 }

 // findKeyAndType returns the start and end position for the type corresponding
 // to key or the point at which to insert the key-value pair if the type
 // wasn't found. The hasExt return value reports whether an -u extension was present.
 // Note: the extensions are typically very small and are likely to contain
 // only one key-type pair.
 func (t Tag) findTypeForKey(key string) (start, sep, end int, hasExt bool) {
 	p := int(t.pExt)
 	if len(key) != 2 || p == len(t.str) || p == 0 {
 		return p, p, p, false
 	}
 	s := t.str

 	// Find the correct extension.
 	for p++; s[p] != 'u'; p++ {
 		if s[p] > 'u' {
 			p--
 			return p, p, p, false
 		}
 		if p = nextExtension(s, p); p == len(s) {
 			return len(s), len(s), len(s), false
 		}
 	}
 	// Proceed to the hyphen following the extension name.
 	p++

 	// curKey is the key currently being processed.
 	curKey := ""

 	// Iterate over keys until we get the end of a section.
 	for {
 		end = p
 		for p++; p < len(s) && s[p] != '-'; p++ {
 		}
 		n := p - end - 1
 		if n <= 2 && curKey == key {
 			if sep < end {
 				sep++
 			}
 			return start, sep, end, true
 		}
 		switch n {
 		case 0, // invalid string
 			1: // next extension
 			return end, end, end, true
 		case 2:
 			// next key
 			curKey = s[end+1 : p]
 			if curKey > key {
 				return end, end, end, true
 			}
 			start = end
 			sep = p
 		}
 	}
 }

 // ParseBase parses a 2- or 3-letter ISO 639 code.
 // It returns a ValueError if s is a well-formed but unknown language identifier
 // or another error if another error occurred.
 func ParseBase(s string) (l Language, err error) {
 	defer func() {
 		if recover() != nil {
 			l = 0
 			err = ErrSyntax
 		}
 	}()

 	if n := len(s); n < 2 || 3 < n {
 		return 0, ErrSyntax
 	}
 	var buf [3]byte
 	return getLangID(buf[:copy(buf[:], s)])
 }

 // ParseScript parses a 4-letter ISO 15924 code.
 // It returns a ValueError if s is a well-formed but unknown script identifier
 // or another error if another error occurred.
 func ParseScript(s string) (scr Script, err error) {
 	defer func() {
 		if recover() != nil {
 			scr = 0
 			err = ErrSyntax
 		}
 	}()

 	if len(s) != 4 {
 		return 0, ErrSyntax
 	}
 	var buf [4]byte
 	return getScriptID(script, buf[:copy(buf[:], s)])
 }

 // EncodeM49 returns the Region for the given UN M.49 code.
 // It returns an error if r is not a valid code.
 func EncodeM49(r int) (Region, error) {
 	return getRegionM49(r)
 }

 // ParseRegion parses a 2- or 3-letter ISO 3166-1 or a UN M.49 code.
 // It returns a ValueError if s is a well-formed but unknown region identifier
 // or another error if another error occurred.
 func ParseRegion(s string) (r Region, err error) {
 	defer func() {
 		if recover() != nil {
 			r = 0
 			err = ErrSyntax
 		}
 	}()

 	if n := len(s); n < 2 || 3 < n {
 		return 0, ErrSyntax
 	}
 	var buf [3]byte
 	return getRegionID(buf[:copy(buf[:], s)])
 }

 // IsCountry returns whether this region is a country or autonomous area. This
 // includes non-standard definitions from CLDR.
 func (r Region) IsCountry() bool {
 	if r == 0 || r.IsGroup() || r.IsPrivateUse() && r != _XK {
 		return false
 	}
 	return true
 }

 // IsGroup returns whether this region defines a collection of regions. This
 // includes non-standard definitions from CLDR.
 func (r Region) IsGroup() bool {
 	if r == 0 {
 		return false
 	}
 	return int(regionInclusion[r]) < len(regionContainment)
 }

 // Contains returns whether Region c is contained by Region r. It returns true
 // if c == r.
 func (r Region) Contains(c Region) bool {
 	if r == c {
 		return true
 	}
 	g := regionInclusion[r]
 	if g >= nRegionGroups {
 		return false
 	}
 	m := regionContainment[g]

 	d := regionInclusion[c]
 	b := regionInclusionBits[d]

 	// A contained country may belong to multiple disjoint groups. Matching any
 	// of these indicates containment. If the contained region is a group, it
 	// must strictly be a subset.
 	if d >= nRegionGroups {
 		return b&m != 0
 	}
 	return b&^m == 0
 }

 var errNoTLD = errors.New("language: region is not a valid ccTLD")

 // TLD returns the country code top-level domain (ccTLD). UK is returned for GB.
 // In all other cases it returns either the region itself or an error.
 //
 // This method may return an error for a region for which there exists a
 // canonical form with a ccTLD. To get that ccTLD canonicalize r first. The
 // region will already be canonicalized it was obtained from a Tag that was
 // obtained using any of the default methods.
 func (r Region) TLD() (Region, error) {
 	// See http://en.wikipedia.org/wiki/Country_code_top-level_domain for the
 	// difference between ISO 3166-1 and IANA ccTLD.
 	if r == _GB {
 		r = _UK
 	}
 	if (r.typ() & ccTLD) == 0 {
 		return 0, errNoTLD
 	}
 	return r, nil
 }

 // Canonicalize returns the region or a possible replacement if the region is
 // deprecated. It will not return a replacement for deprecated regions that
 // are split into multiple regions.
 func (r Region) Canonicalize() Region {
 	if cr := normRegion(r); cr != 0 {
 		return cr
 	}
 	return r
 }

 // Variant represents a registered variant of a language as defined by BCP 47.
 type Variant struct {
 	ID  uint8
 	str string
 }

 // ParseVariant parses and returns a Variant. An error is returned if s is not
 // a valid variant.
 func ParseVariant(s string) (v Variant, err error) {
 	defer func() {
 		if recover() != nil {
 			v = Variant{}
 			err = ErrSyntax
 		}
 	}()

 	s = strings.ToLower(s)
 	if id, ok := variantIndex[s]; ok {
 		return Variant{id, s}, nil
 	}
 	return Variant{}, NewValueError([]byte(s))
 }

 // String returns the string representation of the variant.
 func (v Variant) String() string {
 	return v.str
 }
	// Copyright 2013 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	//go:generate go run gen.go gen_common.go -output tables.go

	package language // import "golang.org/x/text/internal/language"

	// TODO: Remove above NOTE after:
	// - verifying that tables are dropped correctly (most notably matcher tables).

	import (
	"errors"
	"fmt"
	"strings"
	)

	const (
	// maxCoreSize is the maximum size of a BCP 47 tag without variants and
	// extensions. Equals max lang (3) + script (4) + max reg (3) + 2 dashes.
	maxCoreSize = 12

	// max99thPercentileSize is a somewhat arbitrary buffer size that presumably
	// is large enough to hold at least 99% of the BCP 47 tags.
	max99thPercentileSize = 32

	// maxSimpleUExtensionSize is the maximum size of a -u extension with one
	// key-type pair. Equals len("-u-") + key (2) + dash + max value (8).
	maxSimpleUExtensionSize = 14
	)

	// Tag represents a BCP 47 language tag. It is used to specify an instance of a
	// specific language or locale. All language tag values are guaranteed to be
	// well-formed. The zero value of Tag is Und.
	type Tag struct {
	// TODO: the following fields have the form TagTypeID. This name is chosen
	// to allow refactoring the public package without conflicting with its
	// Base, Script, and Region methods. Once the transition is fully completed
	// the ID can be stripped from the name.

	LangID Language
	RegionID Region
	// TODO: we will soon run out of positions for ScriptID. Idea: instead of
	// storing lang, region, and ScriptID codes, store only the compact index and
	// have a lookup table from this code to its expansion. This greatly speeds
	// up table lookup, speed up common variant cases.
	// This will also immediately free up 3 extra bytes. Also, the pVariant
	// field can now be moved to the lookup table, as the compact index uniquely
	// determines the offset of a possible variant.
	ScriptID Script
	pVariant byte // offset in str, includes preceding '-'
	pExt uint16 // offset of first extension, includes preceding '-'

	// str is the string representation of the Tag. It will only be used if the
	// tag has variants or extensions.
	str string
	}

	// Make is a convenience wrapper for Parse that omits the error.
	// In case of an error, a sensible default is returned.
	func Make(s string) Tag {
	t, _ := Parse(s)
	return t
	}

	// Raw returns the raw base language, script and region, without making an
	// attempt to infer their values.
	// TODO: consider removing
	func (t Tag) Raw() (b Language, s Script, r Region) {
	return t.LangID, t.ScriptID, t.RegionID
	}

	// equalTags compares language, script and region subtags only.
	func (t Tag) equalTags(a Tag) bool {
	return t.LangID == a.LangID && t.ScriptID == a.ScriptID && t.RegionID == a.RegionID
	}

	// IsRoot returns true if t is equal to language "und".
	func (t Tag) IsRoot() bool {
	if int(t.pVariant) < len(t.str) {
	return false
	}
	return t.equalTags(Und)
	}

	// IsPrivateUse reports whether the Tag consists solely of an IsPrivateUse use
	// tag.
	func (t Tag) IsPrivateUse() bool {
	return t.str != "" && t.pVariant == 0
	}

	// RemakeString is used to update t.str in case lang, script or region changed.
	// It is assumed that pExt and pVariant still point to the start of the
	// respective parts.
	func (t *Tag) RemakeString() {
	if t.str == "" {
	return
	}
	extra := t.str[t.pVariant:]
	if t.pVariant > 0 {
	extra = extra[1:]
	}
	if t.equalTags(Und) && strings.HasPrefix(extra, "x-") {
	t.str = extra
	t.pVariant = 0
	t.pExt = 0
	return
	}
	var buf [max99thPercentileSize]byte // avoid extra memory allocation in most cases.
	b := buf[:t.genCoreBytes(buf[:])]
	if extra != "" {
	diff := len(b) - int(t.pVariant)
	b = append(b, '-')
	b = append(b, extra...)
	t.pVariant = uint8(int(t.pVariant) + diff)
	t.pExt = uint16(int(t.pExt) + diff)
	} else {
	t.pVariant = uint8(len(b))
	t.pExt = uint16(len(b))
	}
	t.str = string(b)
	}

	// genCoreBytes writes a string for the base languages, script and region tags
	// to the given buffer and returns the number of bytes written. It will never
	// write more than maxCoreSize bytes.
	func (t *Tag) genCoreBytes(buf []byte) int {
	n := t.LangID.StringToBuf(buf[:])
	if t.ScriptID != 0 {
	n += copy(buf[n:], "-")
	n += copy(buf[n:], t.ScriptID.String())
	}
	if t.RegionID != 0 {
	n += copy(buf[n:], "-")
	n += copy(buf[n:], t.RegionID.String())
	}
	return n
	}

	// String returns the canonical string representation of the language tag.
	func (t Tag) String() string {
	if t.str != "" {
	return t.str
	}
	if t.ScriptID == 0 && t.RegionID == 0 {
	return t.LangID.String()
	}
	buf := [maxCoreSize]byte{}
	return string(buf[:t.genCoreBytes(buf[:])])
	}

	// MarshalText implements encoding.TextMarshaler.
	func (t Tag) MarshalText() (text []byte, err error) {
	if t.str != "" {
	text = append(text, t.str...)
	} else if t.ScriptID == 0 && t.RegionID == 0 {
	text = append(text, t.LangID.String()...)
	} else {
	buf := [maxCoreSize]byte{}
	text = buf[:t.genCoreBytes(buf[:])]
	}
	return text, nil
	}

	// UnmarshalText implements encoding.TextUnmarshaler.
	func (t *Tag) UnmarshalText(text []byte) error {
	tag, err := Parse(string(text))
	*t = tag
	return err
	}

	// Variants returns the part of the tag holding all variants or the empty string
	// if there are no variants defined.
	func (t Tag) Variants() string {
	if t.pVariant == 0 {
	return ""
	}
	return t.str[t.pVariant:t.pExt]
	}

	// VariantOrPrivateUseTags returns variants or private use tags.
	func (t Tag) VariantOrPrivateUseTags() string {
	if t.pExt > 0 {
	return t.str[t.pVariant:t.pExt]
	}
	return t.str[t.pVariant:]
	}

	// HasString reports whether this tag defines more than just the raw
	// components.
	func (t Tag) HasString() bool {
	return t.str != ""
	}

	// Parent returns the CLDR parent of t. In CLDR, missing fields in data for a
	// specific language are substituted with fields from the parent language.
	// The parent for a language may change for newer versions of CLDR.
	func (t Tag) Parent() Tag {
	if t.str != "" {
	// Strip the variants and extensions.
	b, s, r := t.Raw()
	t = Tag{LangID: b, ScriptID: s, RegionID: r}
	if t.RegionID == 0 && t.ScriptID != 0 && t.LangID != 0 {
	base, _ := addTags(Tag{LangID: t.LangID})
	if base.ScriptID == t.ScriptID {
	return Tag{LangID: t.LangID}
	}
	}
	return t
	}
	if t.LangID != 0 {
	if t.RegionID != 0 {
	maxScript := t.ScriptID
	if maxScript == 0 {
	max, _ := addTags(t)
	maxScript = max.ScriptID
	}

	for i := range parents {
	if Language(parents[i].lang) == t.LangID && Script(parents[i].maxScript) == maxScript {
	for _, r := range parents[i].fromRegion {
	if Region(r) == t.RegionID {
	return Tag{
	LangID: t.LangID,
	ScriptID: Script(parents[i].script),
	RegionID: Region(parents[i].toRegion),
	}
	}
	}
	}
	}

	// Strip the script if it is the default one.
	base, _ := addTags(Tag{LangID: t.LangID})
	if base.ScriptID != maxScript {
	return Tag{LangID: t.LangID, ScriptID: maxScript}
	}
	return Tag{LangID: t.LangID}
	} else if t.ScriptID != 0 {
	// The parent for an base-script pair with a non-default script is
	// "und" instead of the base language.
	base, _ := addTags(Tag{LangID: t.LangID})
	if base.ScriptID != t.ScriptID {
	return Und
	}
	return Tag{LangID: t.LangID}
	}
	}
	return Und
	}

	// ParseExtension parses s as an extension and returns it on success.
	func ParseExtension(s string) (ext string, err error) {
	defer func() {
	if recover() != nil {
	ext = ""
	err = ErrSyntax
	}
	}()

	scan := makeScannerString(s)
	var end int
	if n := len(scan.token); n != 1 {
	return "", ErrSyntax
	}
	scan.toLower(0, len(scan.b))
	end = parseExtension(&scan)
	if end != len(s) {
	return "", ErrSyntax
	}
	return string(scan.b), nil
	}

	// HasVariants reports whether t has variants.
	func (t Tag) HasVariants() bool {
	return uint16(t.pVariant) < t.pExt
	}

	// HasExtensions reports whether t has extensions.
	func (t Tag) HasExtensions() bool {
	return int(t.pExt) < len(t.str)
	}

	// Extension returns the extension of type x for tag t. It will return
	// false for ok if t does not have the requested extension. The returned
	// extension will be invalid in this case.
	func (t Tag) Extension(x byte) (ext string, ok bool) {
	for i := int(t.pExt); i < len(t.str)-1; {
	var ext string
	i, ext = getExtension(t.str, i)
	if ext[0] == x {
	return ext, true
	}
	}
	return "", false
	}

	// Extensions returns all extensions of t.
	func (t Tag) Extensions() []string {
	e := []string{}
	for i := int(t.pExt); i < len(t.str)-1; {
	var ext string
	i, ext = getExtension(t.str, i)
	e = append(e, ext)
	}
	return e
	}

	// TypeForKey returns the type associated with the given key, where key and type
	// are of the allowed values defined for the Unicode locale extension ('u') in
	// https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
	// TypeForKey will traverse the inheritance chain to get the correct value.
	//
	// If there are multiple types associated with a key, only the first will be
	// returned. If there is no type associated with a key, it returns the empty
	// string.
	func (t Tag) TypeForKey(key string) string {
	if _, start, end, _ := t.findTypeForKey(key); end != start {
	s := t.str[start:end]
	if p := strings.IndexByte(s, '-'); p >= 0 {
	s = s[:p]
	}
	return s
	}
	return ""
	}

	var (
	errPrivateUse = errors.New("cannot set a key on a private use tag")
	errInvalidArguments = errors.New("invalid key or type")
	)

	// SetTypeForKey returns a new Tag with the key set to type, where key and type
	// are of the allowed values defined for the Unicode locale extension ('u') in
	// https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
	// An empty value removes an existing pair with the same key.
	func (t Tag) SetTypeForKey(key, value string) (Tag, error) {
	if t.IsPrivateUse() {
	return t, errPrivateUse
	}
	if len(key) != 2 {
	return t, errInvalidArguments
	}

	// Remove the setting if value is "".
	if value == "" {
	start, sep, end, _ := t.findTypeForKey(key)
	if start != sep {
	// Remove a possible empty extension.
	switch {
	case t.str[start-2] != '-': // has previous elements.
	case end == len(t.str), // end of string
	end+2 < len(t.str) && t.str[end+2] == '-': // end of extension
	start -= 2
	}
	if start == int(t.pVariant) && end == len(t.str) {
	t.str = ""
	t.pVariant, t.pExt = 0, 0
	} else {
	t.str = fmt.Sprintf("%s%s", t.str[:start], t.str[end:])
	}
	}
	return t, nil
	}

	if len(value) < 3 \|\| len(value) > 8 {
	return t, errInvalidArguments
	}

	var (
	buf [maxCoreSize + maxSimpleUExtensionSize]byte
	uStart int // start of the -u extension.
	)

	// Generate the tag string if needed.
	if t.str == "" {
	uStart = t.genCoreBytes(buf[:])
	buf[uStart] = '-'
	uStart++
	}

	// Create new key-type pair and parse it to verify.
	b := buf[uStart:]
	copy(b, "u-")
	copy(b[2:], key)
	b[4] = '-'
	b = b[:5+copy(b[5:], value)]
	scan := makeScanner(b)
	if parseExtensions(&scan); scan.err != nil {
	return t, scan.err
	}

	// Assemble the replacement string.
	if t.str == "" {
	t.pVariant, t.pExt = byte(uStart-1), uint16(uStart-1)
	t.str = string(buf[:uStart+len(b)])
	} else {
	s := t.str
	start, sep, end, hasExt := t.findTypeForKey(key)
	if start == sep {
	if hasExt {
	b = b[2:]
	}
	t.str = fmt.Sprintf("%s-%s%s", s[:sep], b, s[end:])
	} else {
	t.str = fmt.Sprintf("%s-%s%s", s[:start+3], value, s[end:])
	}
	}
	return t, nil
	}

	// findKeyAndType returns the start and end position for the type corresponding
	// to key or the point at which to insert the key-value pair if the type
	// wasn't found. The hasExt return value reports whether an -u extension was present.
	// Note: the extensions are typically very small and are likely to contain
	// only one key-type pair.
	func (t Tag) findTypeForKey(key string) (start, sep, end int, hasExt bool) {
	p := int(t.pExt)
	if len(key) != 2 \|\| p == len(t.str) \|\| p == 0 {
	return p, p, p, false
	}
	s := t.str

	// Find the correct extension.
	for p++; s[p] != 'u'; p++ {
	if s[p] > 'u' {
	p--
	return p, p, p, false
	}
	if p = nextExtension(s, p); p == len(s) {
	return len(s), len(s), len(s), false
	}
	}
	// Proceed to the hyphen following the extension name.
	p++

	// curKey is the key currently being processed.
	curKey := ""

	// Iterate over keys until we get the end of a section.
	for {
	end = p
	for p++; p < len(s) && s[p] != '-'; p++ {
	}
	n := p - end - 1
	if n <= 2 && curKey == key {
	if sep < end {
	sep++
	}
	return start, sep, end, true
	}
	switch n {
	case 0, // invalid string
	1: // next extension
	return end, end, end, true
	case 2:
	// next key
	curKey = s[end+1 : p]
	if curKey > key {
	return end, end, end, true
	}
	start = end
	sep = p
	}
	}
	}

	// ParseBase parses a 2- or 3-letter ISO 639 code.
	// It returns a ValueError if s is a well-formed but unknown language identifier
	// or another error if another error occurred.
	func ParseBase(s string) (l Language, err error) {
	defer func() {
	if recover() != nil {
	l = 0
	err = ErrSyntax
	}
	}()

	if n := len(s); n < 2 \|\| 3 < n {
	return 0, ErrSyntax
	}
	var buf [3]byte
	return getLangID(buf[:copy(buf[:], s)])
	}

	// ParseScript parses a 4-letter ISO 15924 code.
	// It returns a ValueError if s is a well-formed but unknown script identifier
	// or another error if another error occurred.
	func ParseScript(s string) (scr Script, err error) {
	defer func() {
	if recover() != nil {
	scr = 0
	err = ErrSyntax
	}
	}()

	if len(s) != 4 {
	return 0, ErrSyntax
	}
	var buf [4]byte
	return getScriptID(script, buf[:copy(buf[:], s)])
	}

	// EncodeM49 returns the Region for the given UN M.49 code.
	// It returns an error if r is not a valid code.
	func EncodeM49(r int) (Region, error) {
	return getRegionM49(r)
	}

	// ParseRegion parses a 2- or 3-letter ISO 3166-1 or a UN M.49 code.
	// It returns a ValueError if s is a well-formed but unknown region identifier
	// or another error if another error occurred.
	func ParseRegion(s string) (r Region, err error) {
	defer func() {
	if recover() != nil {
	r = 0
	err = ErrSyntax
	}
	}()

	if n := len(s); n < 2 \|\| 3 < n {
	return 0, ErrSyntax
	}
	var buf [3]byte
	return getRegionID(buf[:copy(buf[:], s)])
	}

	// IsCountry returns whether this region is a country or autonomous area. This
	// includes non-standard definitions from CLDR.
	func (r Region) IsCountry() bool {
	if r == 0 \|\| r.IsGroup() \|\| r.IsPrivateUse() && r != _XK {
	return false
	}
	return true
	}

	// IsGroup returns whether this region defines a collection of regions. This
	// includes non-standard definitions from CLDR.
	func (r Region) IsGroup() bool {
	if r == 0 {
	return false
	}
	return int(regionInclusion[r]) < len(regionContainment)
	}

	// Contains returns whether Region c is contained by Region r. It returns true
	// if c == r.
	func (r Region) Contains(c Region) bool {
	if r == c {
	return true
	}
	g := regionInclusion[r]
	if g >= nRegionGroups {
	return false
	}
	m := regionContainment[g]

	d := regionInclusion[c]
	b := regionInclusionBits[d]

	// A contained country may belong to multiple disjoint groups. Matching any
	// of these indicates containment. If the contained region is a group, it
	// must strictly be a subset.
	if d >= nRegionGroups {
	return b&m != 0
	}
	return b&^m == 0
	}

	var errNoTLD = errors.New("language: region is not a valid ccTLD")

	// TLD returns the country code top-level domain (ccTLD). UK is returned for GB.
	// In all other cases it returns either the region itself or an error.
	//
	// This method may return an error for a region for which there exists a
	// canonical form with a ccTLD. To get that ccTLD canonicalize r first. The
	// region will already be canonicalized it was obtained from a Tag that was
	// obtained using any of the default methods.
	func (r Region) TLD() (Region, error) {
	// See http://en.wikipedia.org/wiki/Country_code_top-level_domain for the
	// difference between ISO 3166-1 and IANA ccTLD.
	if r == _GB {
	r = _UK
	}
	if (r.typ() & ccTLD) == 0 {
	return 0, errNoTLD
	}
	return r, nil
	}

	// Canonicalize returns the region or a possible replacement if the region is
	// deprecated. It will not return a replacement for deprecated regions that
	// are split into multiple regions.
	func (r Region) Canonicalize() Region {
	if cr := normRegion(r); cr != 0 {
	return cr
	}
	return r
	}

	// Variant represents a registered variant of a language as defined by BCP 47.
	type Variant struct {
	ID uint8
	str string
	}

	// ParseVariant parses and returns a Variant. An error is returned if s is not
	// a valid variant.
	func ParseVariant(s string) (v Variant, err error) {
	defer func() {
	if recover() != nil {
	v = Variant{}
	err = ErrSyntax
	}
	}()

	s = strings.ToLower(s)
	if id, ok := variantIndex[s]; ok {
	return Variant{id, s}, nil
	}
	return Variant{}, NewValueError([]byte(s))
	}

	// String returns the string representation of the variant.
	func (v Variant) String() string {
	return v.str
	}