src/unicode/letter.go - go - Git at Google

 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 // Package unicode provides data and functions to test some properties of
 // Unicode code points.
 package unicode

 const (
 	MaxRune         = '\U0010FFFF' // Maximum valid Unicode code point.
 	ReplacementChar = '\uFFFD'     // Represents invalid code points.
 	MaxASCII        = '\u007F'     // maximum ASCII value.
 	MaxLatin1       = '\u00FF'     // maximum Latin-1 value.
 )

 // RangeTable defines a set of Unicode code points by listing the ranges of
 // code points within the set. The ranges are listed in two slices
 // to save space: a slice of 16-bit ranges and a slice of 32-bit ranges.
 // The two slices must be in sorted order and non-overlapping.
 // Also, R32 should contain only values >= 0x10000 (1<<16).
 type RangeTable struct {
 	R16         []Range16
 	R32         []Range32
 	LatinOffset int // number of entries in R16 with Hi <= MaxLatin1
 }

 // Range16 represents of a range of 16-bit Unicode code points. The range runs from Lo to Hi
 // inclusive and has the specified stride.
 type Range16 struct {
 	Lo     uint16
 	Hi     uint16
 	Stride uint16
 }

 // Range32 represents of a range of Unicode code points and is used when one or
 // more of the values will not fit in 16 bits. The range runs from Lo to Hi
 // inclusive and has the specified stride. Lo and Hi must always be >= 1<<16.
 type Range32 struct {
 	Lo     uint32
 	Hi     uint32
 	Stride uint32
 }

 // CaseRange represents a range of Unicode code points for simple (one
 // code point to one code point) case conversion.
 // The range runs from Lo to Hi inclusive, with a fixed stride of 1. Deltas
 // are the number to add to the code point to reach the code point for a
 // different case for that character. They may be negative. If zero, it
 // means the character is in the corresponding case. There is a special
 // case representing sequences of alternating corresponding Upper and Lower
 // pairs. It appears with a fixed Delta of
 //
 //	{UpperLower, UpperLower, UpperLower}
 //
 // The constant UpperLower has an otherwise impossible delta value.
 type CaseRange struct {
 	Lo    uint32
 	Hi    uint32
 	Delta d
 }

 // SpecialCase represents language-specific case mappings such as Turkish.
 // Methods of SpecialCase customize (by overriding) the standard mappings.
 type SpecialCase []CaseRange

 // BUG(r): There is no mechanism for full case folding, that is, for
 // characters that involve multiple runes in the input or output.

 // Indices into the Delta arrays inside CaseRanges for case mapping.
 const (
 	UpperCase = iota
 	LowerCase
 	TitleCase
 	MaxCase
 )

 type d [MaxCase]rune // to make the CaseRanges text shorter

 // If the Delta field of a CaseRange is UpperLower, it means
 // this CaseRange represents a sequence of the form (say)
 // Upper Lower Upper Lower.
 const (
 	UpperLower = MaxRune + 1 // (Cannot be a valid delta.)
 )

 // linearMax is the maximum size table for linear search for non-Latin1 rune.
 // Derived by running 'go test -calibrate'.
 const linearMax = 18

 // is16 reports whether r is in the sorted slice of 16-bit ranges.
 func is16(ranges []Range16, r uint16) bool {
 	if len(ranges) <= linearMax || r <= MaxLatin1 {
 		for i := range ranges {
 			range_ := &ranges[i]
 			if r < range_.Lo {
 				return false
 			}
 			if r <= range_.Hi {
 				return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
 			}
 		}
 		return false
 	}

 	// binary search over ranges
 	lo := 0
 	hi := len(ranges)
 	for lo < hi {
 		m := lo + (hi-lo)/2
 		range_ := &ranges[m]
 		if range_.Lo <= r && r <= range_.Hi {
 			return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
 		}
 		if r < range_.Lo {
 			hi = m
 		} else {
 			lo = m + 1
 		}
 	}
 	return false
 }

 // is32 reports whether r is in the sorted slice of 32-bit ranges.
 func is32(ranges []Range32, r uint32) bool {
 	if len(ranges) <= linearMax {
 		for i := range ranges {
 			range_ := &ranges[i]
 			if r < range_.Lo {
 				return false
 			}
 			if r <= range_.Hi {
 				return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
 			}
 		}
 		return false
 	}

 	// binary search over ranges
 	lo := 0
 	hi := len(ranges)
 	for lo < hi {
 		m := lo + (hi-lo)/2
 		range_ := ranges[m]
 		if range_.Lo <= r && r <= range_.Hi {
 			return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
 		}
 		if r < range_.Lo {
 			hi = m
 		} else {
 			lo = m + 1
 		}
 	}
 	return false
 }

 // Is reports whether the rune is in the specified table of ranges.
 func Is(rangeTab *RangeTable, r rune) bool {
 	r16 := rangeTab.R16
 	// Compare as uint32 to correctly handle negative runes.
 	if len(r16) > 0 && uint32(r) <= uint32(r16[len(r16)-1].Hi) {
 		return is16(r16, uint16(r))
 	}
 	r32 := rangeTab.R32
 	if len(r32) > 0 && r >= rune(r32[0].Lo) {
 		return is32(r32, uint32(r))
 	}
 	return false
 }

 func isExcludingLatin(rangeTab *RangeTable, r rune) bool {
 	r16 := rangeTab.R16
 	// Compare as uint32 to correctly handle negative runes.
 	if off := rangeTab.LatinOffset; len(r16) > off && uint32(r) <= uint32(r16[len(r16)-1].Hi) {
 		return is16(r16[off:], uint16(r))
 	}
 	r32 := rangeTab.R32
 	if len(r32) > 0 && r >= rune(r32[0].Lo) {
 		return is32(r32, uint32(r))
 	}
 	return false
 }

 // IsUpper reports whether the rune is an upper case letter.
 func IsUpper(r rune) bool {
 	// See comment in IsGraphic.
 	if uint32(r) <= MaxLatin1 {
 		return properties[uint8(r)]&pLmask == pLu
 	}
 	return isExcludingLatin(Upper, r)
 }

 // IsLower reports whether the rune is a lower case letter.
 func IsLower(r rune) bool {
 	// See comment in IsGraphic.
 	if uint32(r) <= MaxLatin1 {
 		return properties[uint8(r)]&pLmask == pLl
 	}
 	return isExcludingLatin(Lower, r)
 }

 // IsTitle reports whether the rune is a title case letter.
 func IsTitle(r rune) bool {
 	if r <= MaxLatin1 {
 		return false
 	}
 	return isExcludingLatin(Title, r)
 }

 // to maps the rune using the specified case mapping.
 // It additionally reports whether caseRange contained a mapping for r.
 func to(_case int, r rune, caseRange []CaseRange) (mappedRune rune, foundMapping bool) {
 	if _case < 0 || MaxCase <= _case {
 		return ReplacementChar, false // as reasonable an error as any
 	}
 	// binary search over ranges
 	lo := 0
 	hi := len(caseRange)
 	for lo < hi {
 		m := lo + (hi-lo)/2
 		cr := caseRange[m]
 		if rune(cr.Lo) <= r && r <= rune(cr.Hi) {
 			delta := cr.Delta[_case]
 			if delta > MaxRune {
 				// In an Upper-Lower sequence, which always starts with
 				// an UpperCase letter, the real deltas always look like:
 				//	{0, 1, 0}    UpperCase (Lower is next)
 				//	{-1, 0, -1}  LowerCase (Upper, Title are previous)
 				// The characters at even offsets from the beginning of the
 				// sequence are upper case; the ones at odd offsets are lower.
 				// The correct mapping can be done by clearing or setting the low
 				// bit in the sequence offset.
 				// The constants UpperCase and TitleCase are even while LowerCase
 				// is odd so we take the low bit from _case.
 				return rune(cr.Lo) + ((r-rune(cr.Lo))&^1 | rune(_case&1)), true
 			}
 			return r + delta, true
 		}
 		if r < rune(cr.Lo) {
 			hi = m
 		} else {
 			lo = m + 1
 		}
 	}
 	return r, false
 }

 // To maps the rune to the specified case: UpperCase, LowerCase, or TitleCase.
 func To(_case int, r rune) rune {
 	r, _ = to(_case, r, CaseRanges)
 	return r
 }

 // ToUpper maps the rune to upper case.
 func ToUpper(r rune) rune {
 	if r <= MaxASCII {
 		if 'a' <= r && r <= 'z' {
 			r -= 'a' - 'A'
 		}
 		return r
 	}
 	return To(UpperCase, r)
 }

 // ToLower maps the rune to lower case.
 func ToLower(r rune) rune {
 	if r <= MaxASCII {
 		if 'A' <= r && r <= 'Z' {
 			r += 'a' - 'A'
 		}
 		return r
 	}
 	return To(LowerCase, r)
 }

 // ToTitle maps the rune to title case.
 func ToTitle(r rune) rune {
 	if r <= MaxASCII {
 		if 'a' <= r && r <= 'z' { // title case is upper case for ASCII
 			r -= 'a' - 'A'
 		}
 		return r
 	}
 	return To(TitleCase, r)
 }

 // ToUpper maps the rune to upper case giving priority to the special mapping.
 func (special SpecialCase) ToUpper(r rune) rune {
 	r1, hadMapping := to(UpperCase, r, []CaseRange(special))
 	if r1 == r && !hadMapping {
 		r1 = ToUpper(r)
 	}
 	return r1
 }

 // ToTitle maps the rune to title case giving priority to the special mapping.
 func (special SpecialCase) ToTitle(r rune) rune {
 	r1, hadMapping := to(TitleCase, r, []CaseRange(special))
 	if r1 == r && !hadMapping {
 		r1 = ToTitle(r)
 	}
 	return r1
 }

 // ToLower maps the rune to lower case giving priority to the special mapping.
 func (special SpecialCase) ToLower(r rune) rune {
 	r1, hadMapping := to(LowerCase, r, []CaseRange(special))
 	if r1 == r && !hadMapping {
 		r1 = ToLower(r)
 	}
 	return r1
 }

 // caseOrbit is defined in tables.go as []foldPair. Right now all the
 // entries fit in uint16, so use uint16. If that changes, compilation
 // will fail (the constants in the composite literal will not fit in uint16)
 // and the types here can change to uint32.
 type foldPair struct {
 	From uint16
 	To   uint16
 }

 // SimpleFold iterates over Unicode code points equivalent under
 // the Unicode-defined simple case folding. Among the code points
 // equivalent to rune (including rune itself), SimpleFold returns the
 // smallest rune > r if one exists, or else the smallest rune >= 0.
 // If r is not a valid Unicode code point, SimpleFold(r) returns r.
 //
 // For example:
 //
 //	SimpleFold('A') = 'a'
 //	SimpleFold('a') = 'A'
 //
 //	SimpleFold('K') = 'k'
 //	SimpleFold('k') = '\u212A' (Kelvin symbol, K)
 //	SimpleFold('\u212A') = 'K'
 //
 //	SimpleFold('1') = '1'
 //
 //	SimpleFold(-2) = -2
 func SimpleFold(r rune) rune {
 	if r < 0 || r > MaxRune {
 		return r
 	}

 	if int(r) < len(asciiFold) {
 		return rune(asciiFold[r])
 	}

 	// Consult caseOrbit table for special cases.
 	lo := 0
 	hi := len(caseOrbit)
 	for lo < hi {
 		m := lo + (hi-lo)/2
 		if rune(caseOrbit[m].From) < r {
 			lo = m + 1
 		} else {
 			hi = m
 		}
 	}
 	if lo < len(caseOrbit) && rune(caseOrbit[lo].From) == r {
 		return rune(caseOrbit[lo].To)
 	}

 	// No folding specified. This is a one- or two-element
 	// equivalence class containing rune and ToLower(rune)
 	// and ToUpper(rune) if they are different from rune.
 	if l := ToLower(r); l != r {
 		return l
 	}
 	return ToUpper(r)
 }
	// Copyright 2009 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	// Package unicode provides data and functions to test some properties of
	// Unicode code points.
	package unicode

	const (
	MaxRune = '\U0010FFFF' // Maximum valid Unicode code point.
	ReplacementChar = '\uFFFD' // Represents invalid code points.
	MaxASCII = '\u007F' // maximum ASCII value.
	MaxLatin1 = '\u00FF' // maximum Latin-1 value.
	)

	// RangeTable defines a set of Unicode code points by listing the ranges of
	// code points within the set. The ranges are listed in two slices
	// to save space: a slice of 16-bit ranges and a slice of 32-bit ranges.
	// The two slices must be in sorted order and non-overlapping.
	// Also, R32 should contain only values >= 0x10000 (1<<16).
	type RangeTable struct {
	R16 []Range16
	R32 []Range32
	LatinOffset int // number of entries in R16 with Hi <= MaxLatin1
	}

	// Range16 represents of a range of 16-bit Unicode code points. The range runs from Lo to Hi
	// inclusive and has the specified stride.
	type Range16 struct {
	Lo uint16
	Hi uint16
	Stride uint16
	}

	// Range32 represents of a range of Unicode code points and is used when one or
	// more of the values will not fit in 16 bits. The range runs from Lo to Hi
	// inclusive and has the specified stride. Lo and Hi must always be >= 1<<16.
	type Range32 struct {
	Lo uint32
	Hi uint32
	Stride uint32
	}

	// CaseRange represents a range of Unicode code points for simple (one
	// code point to one code point) case conversion.
	// The range runs from Lo to Hi inclusive, with a fixed stride of 1. Deltas
	// are the number to add to the code point to reach the code point for a
	// different case for that character. They may be negative. If zero, it
	// means the character is in the corresponding case. There is a special
	// case representing sequences of alternating corresponding Upper and Lower
	// pairs. It appears with a fixed Delta of
	//
	// {UpperLower, UpperLower, UpperLower}
	//
	// The constant UpperLower has an otherwise impossible delta value.
	type CaseRange struct {
	Lo uint32
	Hi uint32
	Delta d
	}

	// SpecialCase represents language-specific case mappings such as Turkish.
	// Methods of SpecialCase customize (by overriding) the standard mappings.
	type SpecialCase []CaseRange

	// BUG(r): There is no mechanism for full case folding, that is, for
	// characters that involve multiple runes in the input or output.

	// Indices into the Delta arrays inside CaseRanges for case mapping.
	const (
	UpperCase = iota
	LowerCase
	TitleCase
	MaxCase
	)

	type d [MaxCase]rune // to make the CaseRanges text shorter

	// If the Delta field of a CaseRange is UpperLower, it means
	// this CaseRange represents a sequence of the form (say)
	// Upper Lower Upper Lower.
	const (
	UpperLower = MaxRune + 1 // (Cannot be a valid delta.)
	)

	// linearMax is the maximum size table for linear search for non-Latin1 rune.
	// Derived by running 'go test -calibrate'.
	const linearMax = 18

	// is16 reports whether r is in the sorted slice of 16-bit ranges.
	func is16(ranges []Range16, r uint16) bool {
	if len(ranges) <= linearMax \|\| r <= MaxLatin1 {
	for i := range ranges {
	range_ := &ranges[i]
	if r < range_.Lo {
	return false
	}
	if r <= range_.Hi {
	return range_.Stride == 1 \|\| (r-range_.Lo)%range_.Stride == 0
	}
	}
	return false
	}

	// binary search over ranges
	lo := 0
	hi := len(ranges)
	for lo < hi {
	m := lo + (hi-lo)/2
	range_ := &ranges[m]
	if range_.Lo <= r && r <= range_.Hi {
	return range_.Stride == 1 \|\| (r-range_.Lo)%range_.Stride == 0
	}
	if r < range_.Lo {
	hi = m
	} else {
	lo = m + 1
	}
	}
	return false
	}

	// is32 reports whether r is in the sorted slice of 32-bit ranges.
	func is32(ranges []Range32, r uint32) bool {
	if len(ranges) <= linearMax {
	for i := range ranges {
	range_ := &ranges[i]
	if r < range_.Lo {
	return false
	}
	if r <= range_.Hi {
	return range_.Stride == 1 \|\| (r-range_.Lo)%range_.Stride == 0
	}
	}
	return false
	}

	// binary search over ranges
	lo := 0
	hi := len(ranges)
	for lo < hi {
	m := lo + (hi-lo)/2
	range_ := ranges[m]
	if range_.Lo <= r && r <= range_.Hi {
	return range_.Stride == 1 \|\| (r-range_.Lo)%range_.Stride == 0
	}
	if r < range_.Lo {
	hi = m
	} else {
	lo = m + 1
	}
	}
	return false
	}

	// Is reports whether the rune is in the specified table of ranges.
	func Is(rangeTab *RangeTable, r rune) bool {
	r16 := rangeTab.R16
	// Compare as uint32 to correctly handle negative runes.
	if len(r16) > 0 && uint32(r) <= uint32(r16[len(r16)-1].Hi) {
	return is16(r16, uint16(r))
	}
	r32 := rangeTab.R32
	if len(r32) > 0 && r >= rune(r32[0].Lo) {
	return is32(r32, uint32(r))
	}
	return false
	}

	func isExcludingLatin(rangeTab *RangeTable, r rune) bool {
	r16 := rangeTab.R16
	// Compare as uint32 to correctly handle negative runes.
	if off := rangeTab.LatinOffset; len(r16) > off && uint32(r) <= uint32(r16[len(r16)-1].Hi) {
	return is16(r16[off:], uint16(r))
	}
	r32 := rangeTab.R32
	if len(r32) > 0 && r >= rune(r32[0].Lo) {
	return is32(r32, uint32(r))
	}
	return false
	}

	// IsUpper reports whether the rune is an upper case letter.
	func IsUpper(r rune) bool {
	// See comment in IsGraphic.
	if uint32(r) <= MaxLatin1 {
	return properties[uint8(r)]&pLmask == pLu
	}
	return isExcludingLatin(Upper, r)
	}

	// IsLower reports whether the rune is a lower case letter.
	func IsLower(r rune) bool {
	// See comment in IsGraphic.
	if uint32(r) <= MaxLatin1 {
	return properties[uint8(r)]&pLmask == pLl
	}
	return isExcludingLatin(Lower, r)
	}

	// IsTitle reports whether the rune is a title case letter.
	func IsTitle(r rune) bool {
	if r <= MaxLatin1 {
	return false
	}
	return isExcludingLatin(Title, r)
	}

	// to maps the rune using the specified case mapping.
	// It additionally reports whether caseRange contained a mapping for r.
	func to(_case int, r rune, caseRange []CaseRange) (mappedRune rune, foundMapping bool) {
	if _case < 0 \|\| MaxCase <= _case {
	return ReplacementChar, false // as reasonable an error as any
	}
	// binary search over ranges
	lo := 0
	hi := len(caseRange)
	for lo < hi {
	m := lo + (hi-lo)/2
	cr := caseRange[m]
	if rune(cr.Lo) <= r && r <= rune(cr.Hi) {
	delta := cr.Delta[_case]
	if delta > MaxRune {
	// In an Upper-Lower sequence, which always starts with
	// an UpperCase letter, the real deltas always look like:
	// {0, 1, 0} UpperCase (Lower is next)
	// {-1, 0, -1} LowerCase (Upper, Title are previous)
	// The characters at even offsets from the beginning of the
	// sequence are upper case; the ones at odd offsets are lower.
	// The correct mapping can be done by clearing or setting the low
	// bit in the sequence offset.
	// The constants UpperCase and TitleCase are even while LowerCase
	// is odd so we take the low bit from _case.
	return rune(cr.Lo) + ((r-rune(cr.Lo))&^1 \| rune(_case&1)), true
	}
	return r + delta, true
	}
	if r < rune(cr.Lo) {
	hi = m
	} else {
	lo = m + 1
	}
	}
	return r, false
	}

	// To maps the rune to the specified case: UpperCase, LowerCase, or TitleCase.
	func To(_case int, r rune) rune {
	r, _ = to(_case, r, CaseRanges)
	return r
	}

	// ToUpper maps the rune to upper case.
	func ToUpper(r rune) rune {
	if r <= MaxASCII {
	if 'a' <= r && r <= 'z' {
	r -= 'a' - 'A'
	}
	return r
	}
	return To(UpperCase, r)
	}

	// ToLower maps the rune to lower case.
	func ToLower(r rune) rune {
	if r <= MaxASCII {
	if 'A' <= r && r <= 'Z' {
	r += 'a' - 'A'
	}
	return r
	}
	return To(LowerCase, r)
	}

	// ToTitle maps the rune to title case.
	func ToTitle(r rune) rune {
	if r <= MaxASCII {
	if 'a' <= r && r <= 'z' { // title case is upper case for ASCII
	r -= 'a' - 'A'
	}
	return r
	}
	return To(TitleCase, r)
	}

	// ToUpper maps the rune to upper case giving priority to the special mapping.
	func (special SpecialCase) ToUpper(r rune) rune {
	r1, hadMapping := to(UpperCase, r, []CaseRange(special))
	if r1 == r && !hadMapping {
	r1 = ToUpper(r)
	}
	return r1
	}

	// ToTitle maps the rune to title case giving priority to the special mapping.
	func (special SpecialCase) ToTitle(r rune) rune {
	r1, hadMapping := to(TitleCase, r, []CaseRange(special))
	if r1 == r && !hadMapping {
	r1 = ToTitle(r)
	}
	return r1
	}

	// ToLower maps the rune to lower case giving priority to the special mapping.
	func (special SpecialCase) ToLower(r rune) rune {
	r1, hadMapping := to(LowerCase, r, []CaseRange(special))
	if r1 == r && !hadMapping {
	r1 = ToLower(r)
	}
	return r1
	}

	// caseOrbit is defined in tables.go as []foldPair. Right now all the
	// entries fit in uint16, so use uint16. If that changes, compilation
	// will fail (the constants in the composite literal will not fit in uint16)
	// and the types here can change to uint32.
	type foldPair struct {
	From uint16
	To uint16
	}

	// SimpleFold iterates over Unicode code points equivalent under
	// the Unicode-defined simple case folding. Among the code points
	// equivalent to rune (including rune itself), SimpleFold returns the
	// smallest rune > r if one exists, or else the smallest rune >= 0.
	// If r is not a valid Unicode code point, SimpleFold(r) returns r.
	//
	// For example:
	//
	// SimpleFold('A') = 'a'
	// SimpleFold('a') = 'A'
	//
	// SimpleFold('K') = 'k'
	// SimpleFold('k') = '\u212A' (Kelvin symbol, K)
	// SimpleFold('\u212A') = 'K'
	//
	// SimpleFold('1') = '1'
	//
	// SimpleFold(-2) = -2
	func SimpleFold(r rune) rune {
	if r < 0 \|\| r > MaxRune {
	return r
	}

	if int(r) < len(asciiFold) {
	return rune(asciiFold[r])
	}

	// Consult caseOrbit table for special cases.
	lo := 0
	hi := len(caseOrbit)
	for lo < hi {
	m := lo + (hi-lo)/2
	if rune(caseOrbit[m].From) < r {
	lo = m + 1
	} else {
	hi = m
	}
	}
	if lo < len(caseOrbit) && rune(caseOrbit[lo].From) == r {
	return rune(caseOrbit[lo].To)
	}

	// No folding specified. This is a one- or two-element
	// equivalence class containing rune and ToLower(rune)
	// and ToUpper(rune) if they are different from rune.
	if l := ToLower(r); l != r {
	return l
	}
	return ToUpper(r)
	}