runes: new runes package.
Added Map, Remove, and If transforms and Set interface.
Change-Id: Ia11d06efdf8186028bc89e0090d8c9d9db13f0ea
Reviewed-on: https://go-review.googlesource.com/8336
Reviewed-by: Nigel Tao <nigeltao@golang.org>
diff --git a/runes/cond.go b/runes/cond.go
new file mode 100644
index 0000000..1a97872
--- /dev/null
+++ b/runes/cond.go
@@ -0,0 +1,126 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runes
+
+import (
+ "unicode/utf8"
+
+ "golang.org/x/text/transform"
+)
+
+// Note: below we pass invalid UTF-8 to the tIn and tNotIn transformers as is.
+// This is done for various reasons:
+// - To retain the semantics of the Nop transformer: if input is passed to a Nop
+// one would expect it to be unchanged.
+// - It would be very expensive to pass a converted RuneError to a transformer:
+// a transformer might need more source bytes after RuneError, meaning that
+// the only way to pass it safely is to create a new buffer and manage the
+// intermingling of RuneErrors and normal input.
+// - Many transformers leave ill-formed UTF-8 as is, so this is not
+// inconsistent. Generally ill-formed UTF-8 is only replaced if it is a
+// logical consequence of the operation (as for Map) or if it otherwise would
+// pose security concerns (as for Remove).
+// - An alternative would be to return an error on ill-formed UTF-8, but this
+// would be inconsistent with other operations.
+
+// If returns a transformer that applies tIn to consecutive runes for which
+// s.Contains(r) and tNotIn to consecutive runes for which !s.Contains(r). Reset
+// is called on tIn and tNotIn at the start of each run. A Nop transformer will
+// substitute a nil value passed to tIn or tNotIn. Invalid UTF-8 is translated
+// to RuneError to determine which transformer to apply, but is passed as is to
+// the respective transformer.
+func If(s Set, tIn, tNotIn transform.Transformer) transform.Transformer {
+ if tIn == nil && tNotIn == nil {
+ return transform.Nop
+ }
+ if tIn == nil {
+ tIn = transform.Nop
+ }
+ if tNotIn == nil {
+ tNotIn = transform.Nop
+ }
+ a := &cond{
+ tIn: tIn,
+ tNotIn: tNotIn,
+ f: s.Contains,
+ }
+ a.Reset()
+ return a
+}
+
+type cond struct {
+ tIn, tNotIn transform.Transformer
+ f func(rune) bool
+ check func(rune) bool // current check to perform
+ t transform.Transformer // current transformer to use
+}
+
+// Reset implements transform.Transformer.
+func (t *cond) Reset() {
+ t.check = t.is
+ t.t = t.tIn
+ t.t.Reset() // notIn will be reset on first usage.
+}
+
+func (t *cond) is(r rune) bool {
+ if t.f(r) {
+ return true
+ }
+ t.check = t.isNot
+ t.t = t.tNotIn
+ t.tNotIn.Reset()
+ return false
+}
+
+func (t *cond) isNot(r rune) bool {
+ if !t.f(r) {
+ return true
+ }
+ t.check = t.is
+ t.t = t.tIn
+ t.tIn.Reset()
+ return false
+}
+
+func (t *cond) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+ p := 0
+ for nSrc < len(src) && err == nil {
+ // Don't process too much at a time, as the work might be wasted if the
+ // destination buffer isn't large enough to hold the result or a
+ // transform returns an error early.
+ const maxChunk = 4096
+ max := len(src)
+ if n := nSrc + maxChunk; n < len(src) {
+ max = n
+ }
+ atEnd := false
+ size := 0
+ current := t.t
+ for ; p < max; p += size {
+ var r rune
+ r, size = utf8.DecodeRune(src[p:])
+ if r == utf8.RuneError && size == 1 {
+ if !atEOF && !utf8.FullRune(src[p:]) {
+ err = transform.ErrShortSrc
+ break
+ }
+ }
+ if !t.check(r) {
+ // The next rune will be the start of a new run.
+ atEnd = true
+ break
+ }
+ }
+ nDst2, nSrc2, err2 := current.Transform(dst[nDst:], src[nSrc:p], atEnd || (atEOF && p == len(src)))
+ nDst += nDst2
+ nSrc += nSrc2
+ if err2 != nil {
+ return nDst, nSrc, err2
+ }
+ // At this point either err != nil or t.check will pass for the rune at p.
+ p = nSrc + size
+ }
+ return nDst, nSrc, err
+}
diff --git a/runes/cond_test.go b/runes/cond_test.go
new file mode 100644
index 0000000..175bd2b
--- /dev/null
+++ b/runes/cond_test.go
@@ -0,0 +1,233 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runes
+
+import (
+ "strings"
+ "testing"
+ "unicode"
+
+ "golang.org/x/text/cases"
+ "golang.org/x/text/language"
+ "golang.org/x/text/transform"
+)
+
+var (
+ toUpper = cases.Upper(language.Und)
+ toLower = cases.Lower(language.Und)
+)
+
+func TestPredicate(t *testing.T) {
+ testConditional(t, func(rt *unicode.RangeTable, t, f transform.Transformer) transform.Transformer {
+ return If(Predicate(func(r rune) bool {
+ return unicode.Is(rt, r)
+ }), t, f)
+ })
+}
+
+func TestIn(t *testing.T) {
+ testConditional(t, func(rt *unicode.RangeTable, t, f transform.Transformer) transform.Transformer {
+ return If(In(rt), t, f)
+ })
+}
+
+func TestNotIn(t *testing.T) {
+ testConditional(t, func(rt *unicode.RangeTable, t, f transform.Transformer) transform.Transformer {
+ return If(NotIn(rt), f, t)
+ })
+}
+
+func testConditional(t *testing.T, f func(rt *unicode.RangeTable, t, f transform.Transformer) transform.Transformer) {
+ lower := f(unicode.Latin, toLower, toLower)
+
+ for i, tt := range []transformTest{{
+ desc: "empty",
+ szDst: large,
+ atEOF: true,
+ in: "",
+ out: "",
+ outFull: "",
+ t: lower,
+ }, {
+ desc: "small",
+ szDst: 1,
+ atEOF: true,
+ in: "B",
+ out: "b",
+ outFull: "b",
+ t: lower,
+ }, {
+ desc: "short dst",
+ szDst: 2,
+ atEOF: true,
+ in: "AAA",
+ out: "aa",
+ outFull: "aaa",
+ err: transform.ErrShortDst,
+ t: lower,
+ }, {
+ desc: "short dst writing error",
+ szDst: 1,
+ atEOF: false,
+ in: "A\x80",
+ out: "a",
+ outFull: "a\x80",
+ err: transform.ErrShortDst,
+ t: lower,
+ }, {
+ desc: "short dst writing incomplete rune",
+ szDst: 2,
+ atEOF: true,
+ in: "Σ\xc0",
+ out: "Σ",
+ outFull: "Σ\xc0",
+ err: transform.ErrShortDst,
+ t: f(unicode.Latin, toLower, nil),
+ }, {
+ desc: "short dst, longer",
+ szDst: 5,
+ atEOF: true,
+ in: "Hellø",
+ out: "Hell",
+ outFull: "Hellø",
+ err: transform.ErrShortDst,
+ // idem is used to test short buffers by forcing processing of full-rune increments.
+ t: f(unicode.Latin, Map(idem), nil),
+ }, {
+ desc: "short dst, longer, writing error",
+ szDst: 6,
+ atEOF: false,
+ in: "\x80Hello\x80",
+ out: "\x80Hello",
+ outFull: "\x80Hello\x80",
+ err: transform.ErrShortDst,
+ t: f(unicode.Latin, Map(idem), nil),
+ }, {
+ desc: "short src",
+ szDst: 2,
+ atEOF: false,
+ in: "A\xc0",
+ out: "a",
+ outFull: "a\xc0",
+ err: transform.ErrShortSrc,
+ t: lower,
+ }, {
+ desc: "invalid input, atEOF",
+ szDst: large,
+ atEOF: true,
+ in: "\x80",
+ out: "\x80",
+ outFull: "\x80",
+ t: lower,
+ }, {
+ desc: "invalid input, !atEOF",
+ szDst: large,
+ atEOF: false,
+ in: "\x80",
+ out: "\x80",
+ outFull: "\x80",
+ t: lower,
+ }, {
+ desc: "invalid input, incomplete rune atEOF",
+ szDst: large,
+ atEOF: true,
+ in: "\xc0",
+ out: "\xc0",
+ outFull: "\xc0",
+ t: lower,
+ }, {
+ desc: "nop",
+ szDst: large,
+ atEOF: true,
+ in: "Hello World!",
+ out: "Hello World!",
+ outFull: "Hello World!",
+ t: f(unicode.Latin, nil, nil),
+ }, {
+ desc: "nop in",
+ szDst: large,
+ atEOF: true,
+ in: "THIS IS α ΤΕΣΤ",
+ out: "this is α ΤΕΣΤ",
+ outFull: "this is α ΤΕΣΤ",
+ t: f(unicode.Greek, nil, toLower),
+ }, {
+ desc: "nop not in",
+ szDst: large,
+ atEOF: true,
+ in: "THIS IS α ΤΕΣΤ",
+ out: "this is α ΤΕΣΤ",
+ outFull: "this is α ΤΕΣΤ",
+ t: f(unicode.Latin, toLower, nil),
+ }, {
+ desc: "pass atEOF is true when at end",
+ szDst: large,
+ atEOF: true,
+ in: "hello",
+ out: "HELLO",
+ outFull: "HELLO",
+ t: f(unicode.Latin, upperAtEOF{}, nil),
+ }, {
+ desc: "pass atEOF is true when at end of segment",
+ szDst: large,
+ atEOF: true,
+ in: "hello ",
+ out: "HELLO ",
+ outFull: "HELLO ",
+ t: f(unicode.Latin, upperAtEOF{}, nil),
+ }, {
+ desc: "don't pass atEOF is true when atEOF is false",
+ szDst: large,
+ atEOF: false,
+ in: "hello",
+ out: "",
+ outFull: "HELLO",
+ t: f(unicode.Latin, upperAtEOF{}, nil),
+ err: transform.ErrShortSrc,
+ }, {
+ desc: "large input ASCII",
+ szDst: 12000,
+ atEOF: false,
+ in: strings.Repeat("HELLO", 2000),
+ out: strings.Repeat("hello", 2000),
+ outFull: strings.Repeat("hello", 2000),
+ t: lower,
+ err: nil,
+ }, {
+ desc: "large input non-ASCII",
+ szDst: 12000,
+ atEOF: false,
+ in: strings.Repeat("\u3333", 2000),
+ out: strings.Repeat("\u3333", 2000),
+ outFull: strings.Repeat("\u3333", 2000),
+ t: lower,
+ err: nil,
+ }} {
+ tt.check(t, i)
+ }
+}
+
+// upperAtEOF is a strange Transformer that converts text to uppercase, but only
+// if atEOF is true.
+type upperAtEOF struct{ transform.NopResetter }
+
+func (upperAtEOF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+ if !atEOF {
+ return 0, 0, transform.ErrShortSrc
+ }
+ return toUpper.Transform(dst, src, atEOF)
+}
+
+func BenchmarkConditional(b *testing.B) {
+ dst := make([]byte, len(input))
+ src := []byte(input)
+
+ r := If(In(unicode.Hangul), transform.Nop, transform.Nop)
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ r.Transform(dst, src, true)
+ }
+}
diff --git a/runes/example_test.go b/runes/example_test.go
new file mode 100644
index 0000000..a60bfd9
--- /dev/null
+++ b/runes/example_test.go
@@ -0,0 +1,60 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runes_test
+
+import (
+ "fmt"
+ "unicode"
+
+ "golang.org/x/text/runes"
+ "golang.org/x/text/transform"
+ "golang.org/x/text/unicode/norm"
+ "golang.org/x/text/width"
+)
+
+func ExampleRemove() {
+ t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)
+ s, _, _ := transform.String(t, "résumé")
+ fmt.Println(s)
+
+ // Output:
+ // resume
+}
+
+func ExampleMap() {
+ replaceHyphens := runes.Map(func(r rune) rune {
+ if unicode.Is(unicode.Hyphen, r) {
+ return '|'
+ }
+ return r
+ })
+ s, _, _ := transform.String(replaceHyphens, "a-b‐c⸗d﹣e")
+ fmt.Println(s)
+
+ // Output:
+ // a|b|c|d|e
+}
+
+func ExampleIn() {
+ // Convert Latin characters to their canonical form, while keeping other
+ // width distinctions.
+ t := runes.If(runes.In(unicode.Latin), width.Fold, nil)
+ s, _, _ := transform.String(t, "ｱﾙｱﾉﾘｳ tech / アルアノリウ ｔｅｃｈ")
+ fmt.Println(s)
+
+ // Output:
+ // ｱﾙｱﾉﾘｳ tech / アルアノリウ tech
+}
+
+func ExampleIf() {
+ // Widen everything but ASCII.
+ isASCII := func(r rune) bool { return r <= unicode.MaxASCII }
+ t := runes.If(runes.Predicate(isASCII), nil, width.Widen)
+ s, _, _ := transform.String(t, "ｱﾙｱﾉﾘｳ tech / 中國 / 5₩")
+ fmt.Println(s)
+
+ // Output:
+ // アルアノリウ tech / 中國 / 5￦
+}
diff --git a/runes/runes.go b/runes/runes.go
new file mode 100644
index 0000000..6daa55b
--- /dev/null
+++ b/runes/runes.go
@@ -0,0 +1,196 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package runes provide transforms for UTF-8 encoded text.
+package runes
+
+import (
+ "unicode"
+ "unicode/utf8"
+
+ "golang.org/x/text/transform"
+)
+
+// A Set is a collection of runes.
+type Set interface {
+ // Contains returns true if r is contained in the set.
+ Contains(r rune) bool
+}
+
+type setFunc func(rune) bool
+
+func (s setFunc) Contains(r rune) bool {
+ return s(r)
+}
+
+// Note: using funcs here instead of wrapping types result in cleaner
+// documentation and a smaller API.
+
+// In creates a Set with a Contains method that returns true for all runes in
+// the given RangeTable.
+func In(rt *unicode.RangeTable) Set {
+ return setFunc(func(r rune) bool { return unicode.Is(rt, r) })
+}
+
+// In creates a Set with a Contains method that returns true for all runes not
+// in the given RangeTable.
+func NotIn(rt *unicode.RangeTable) Set {
+ return setFunc(func(r rune) bool { return !unicode.Is(rt, r) })
+}
+
+// Predicate creates a Set with a Contains method that returns f(r).
+func Predicate(f func(rune) bool) Set {
+ return setFunc(f)
+}
+
+// TODO:
+// - Copy: copying strings and bytes in whole-rune units.
+// - Validation (maybe)
+// - Well-formed-ness (maybe)
+
+const runeErrorString = string(utf8.RuneError)
+
+// Remove returns a Transformer that removes runes r for which s.Contains(r).
+// Illegal input bytes are replaced by RuneError before being passed to f.
+func Remove(s Set) transform.Transformer {
+ if f, ok := s.(setFunc); ok {
+ // This little trick cuts the running time of BenchmarkRemove for sets
+ // created by Predicate roughly in half.
+ // TODO: special-case RangeTables as well.
+ return remove(f)
+ }
+ return remove(s.Contains)
+}
+
+// TODO: remove transform.RemoveFunc.
+
+type remove func(r rune) bool
+
+func (remove) Reset() {}
+
+// Transform implements transform.Transformer.
+func (t remove) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+ for r, size := rune(0), 0; nSrc < len(src); {
+ if r = rune(src[nSrc]); r < utf8.RuneSelf {
+ size = 1
+ } else {
+ r, size = utf8.DecodeRune(src[nSrc:])
+
+ if size == 1 {
+ // Invalid rune.
+ if !atEOF && !utf8.FullRune(src[nSrc:]) {
+ err = transform.ErrShortSrc
+ break
+ }
+ // We replace illegal bytes with RuneError. Not doing so might
+ // otherwise turn a sequence of invalid UTF-8 into valid UTF-8.
+ // The resulting byte sequence may subsequently contain runes
+ // for which t(r) is true that were passed unnoticed.
+ if !t(utf8.RuneError) {
+ if nDst+3 > len(dst) {
+ err = transform.ErrShortDst
+ break
+ }
+ dst[nDst+0] = runeErrorString[0]
+ dst[nDst+1] = runeErrorString[1]
+ dst[nDst+2] = runeErrorString[2]
+ nDst += 3
+ }
+ nSrc++
+ continue
+ }
+ }
+
+ if t(r) {
+ nSrc += size
+ continue
+ }
+ if nDst+size > len(dst) {
+ err = transform.ErrShortDst
+ break
+ }
+ for i := 0; i < size; i++ {
+ dst[nDst] = src[nSrc]
+ nDst++
+ nSrc++
+ }
+ }
+ return
+}
+
+// Map returns a Transformer that maps the runes in the input using the given
+// mapping. Illegal bytes in the input are converted to utf8.RuneError before
+// being passed to the mapping func.
+func Map(mapping func(rune) rune) transform.Transformer {
+ return mapper(mapping)
+}
+
+type mapper func(rune) rune
+
+func (mapper) Reset() {}
+
+// Transform implements transform.Transformer.
+func (t mapper) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+ var replacement rune
+ var b [utf8.UTFMax]byte
+
+ for r, size := rune(0), 0; nSrc < len(src); {
+ if r = rune(src[nSrc]); r < utf8.RuneSelf {
+ if replacement = t(r); replacement < utf8.RuneSelf {
+ if nDst == len(dst) {
+ err = transform.ErrShortDst
+ break
+ }
+ dst[nDst] = byte(replacement)
+ nDst++
+ nSrc++
+ continue
+ }
+ size = 1
+ } else if r, size = utf8.DecodeRune(src[nSrc:]); size == 1 {
+ // Invalid rune.
+ if !atEOF && !utf8.FullRune(src[nSrc:]) {
+ err = transform.ErrShortSrc
+ break
+ }
+
+ if replacement = t(utf8.RuneError); replacement == utf8.RuneError {
+ if nDst+3 > len(dst) {
+ err = transform.ErrShortDst
+ break
+ }
+ dst[nDst+0] = runeErrorString[0]
+ dst[nDst+1] = runeErrorString[1]
+ dst[nDst+2] = runeErrorString[2]
+ nDst += 3
+ nSrc++
+ continue
+ }
+ } else if replacement = t(r); replacement == r {
+ if nDst+size > len(dst) {
+ err = transform.ErrShortDst
+ break
+ }
+ for i := 0; i < size; i++ {
+ dst[nDst] = src[nSrc]
+ nDst++
+ nSrc++
+ }
+ continue
+ }
+
+ n := utf8.EncodeRune(b[:], replacement)
+
+ if nDst+n > len(dst) {
+ err = transform.ErrShortDst
+ break
+ }
+ for i := 0; i < n; i++ {
+ dst[nDst] = b[i]
+ nDst++
+ }
+ nSrc += size
+ }
+ return
+}
diff --git a/runes/runes_test.go b/runes/runes_test.go
new file mode 100644
index 0000000..4fbc333
--- /dev/null
+++ b/runes/runes_test.go
@@ -0,0 +1,451 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runes
+
+import (
+ "strings"
+ "testing"
+ "unicode/utf8"
+
+ "golang.org/x/text/transform"
+)
+
+type transformTest struct {
+ desc string
+ szDst int
+ atEOF bool
+ repl string
+ in string
+ out string // result string of first call to Transform
+ outFull string // transform of entire input string
+ err error
+
+ t transform.Transformer
+}
+
+const large = 10240
+
+func (tt *transformTest) check(t *testing.T, i int) {
+ if tt.t == nil {
+ return
+ }
+ dst := make([]byte, tt.szDst)
+ src := []byte(tt.in)
+ nDst, nSrc, err := tt.t.Transform(dst, src, tt.atEOF)
+ if err != tt.err {
+ t.Errorf("%d:%s:error: got %v; want %v", i, tt.desc, err, tt.err)
+ }
+ if got := string(dst[:nDst]); got != tt.out {
+ t.Errorf("%d:%s:out: got %q; want %q", i, tt.desc, got, tt.out)
+ }
+
+ // Calls tt.t.Transform for the remainder of the input. We use this to test
+ // the nSrc return value.
+ out := make([]byte, large)
+ n := copy(out, dst[:nDst])
+ nDst, _, _ = tt.t.Transform(out[n:], src[nSrc:], true)
+ if got, want := string(out[:n+nDst]), tt.outFull; got != want {
+ t.Errorf("%d:%s:outFull: got %q; want %q", i, tt.desc, got, want)
+ }
+}
+
+func idem(r rune) rune { return r }
+
+func TestMap(t *testing.T) {
+ runes := []rune{'a', 'ç', '中', '\U00012345', 'a'}
+ // Default mapper used for this test.
+ rotate := Map(func(r rune) rune {
+ for i, m := range runes {
+ if m == r {
+ return runes[i+1]
+ }
+ }
+ return r
+ })
+
+ for i, tt := range []transformTest{{
+ desc: "empty",
+ szDst: large,
+ atEOF: true,
+ in: "",
+ out: "",
+ outFull: "",
+ t: rotate,
+ }, {
+ desc: "no change",
+ szDst: 1,
+ atEOF: true,
+ in: "b",
+ out: "b",
+ outFull: "b",
+ t: rotate,
+ }, {
+ desc: "short dst",
+ szDst: 2,
+ atEOF: true,
+ in: "aaaa",
+ out: "ç",
+ outFull: "çççç",
+ err: transform.ErrShortDst,
+ t: rotate,
+ }, {
+ desc: "short dst ascii, no change",
+ szDst: 2,
+ atEOF: true,
+ in: "bbb",
+ out: "bb",
+ outFull: "bbb",
+ err: transform.ErrShortDst,
+ t: rotate,
+ }, {
+ desc: "short dst writing error",
+ szDst: 2,
+ atEOF: false,
+ in: "a\x80",
+ out: "ç",
+ outFull: "ç\ufffd",
+ err: transform.ErrShortDst,
+ t: rotate,
+ }, {
+ desc: "short dst writing incomplete rune",
+ szDst: 2,
+ atEOF: true,
+ in: "a\xc0",
+ out: "ç",
+ outFull: "ç\ufffd",
+ err: transform.ErrShortDst,
+ t: rotate,
+ }, {
+ desc: "short dst, longer",
+ szDst: 5,
+ atEOF: true,
+ in: "Hellø",
+ out: "Hell",
+ outFull: "Hellø",
+ err: transform.ErrShortDst,
+ t: rotate,
+ }, {
+ desc: "short dst, single",
+ szDst: 1,
+ atEOF: false,
+ in: "ø",
+ out: "",
+ outFull: "ø",
+ err: transform.ErrShortDst,
+ t: Map(idem),
+ }, {
+ desc: "short dst, longer, writing error",
+ szDst: 8,
+ atEOF: false,
+ in: "\x80Hello\x80",
+ out: "\ufffdHello",
+ outFull: "\ufffdHello\ufffd",
+ err: transform.ErrShortDst,
+ t: rotate,
+ }, {
+ desc: "short src",
+ szDst: 2,
+ atEOF: false,
+ in: "a\xc0",
+ out: "ç",
+ outFull: "ç\ufffd",
+ err: transform.ErrShortSrc,
+ t: rotate,
+ }, {
+ desc: "invalid input, atEOF",
+ szDst: large,
+ atEOF: true,
+ in: "\x80",
+ out: "\ufffd",
+ outFull: "\ufffd",
+ t: rotate,
+ }, {
+ desc: "invalid input, !atEOF",
+ szDst: large,
+ atEOF: false,
+ in: "\x80",
+ out: "\ufffd",
+ outFull: "\ufffd",
+ t: rotate,
+ }, {
+ desc: "invalid input, incomplete rune atEOF",
+ szDst: large,
+ atEOF: true,
+ in: "\xc0",
+ out: "\ufffd",
+ outFull: "\ufffd",
+ t: rotate,
+ }, {
+ desc: "misc correct",
+ szDst: large,
+ atEOF: true,
+ in: "a\U00012345 ç!",
+ out: "ça 中!",
+ outFull: "ça 中!",
+ t: rotate,
+ }, {
+ desc: "misc correct and invalid",
+ szDst: large,
+ atEOF: true,
+ in: "Hello\x80 w\x80orl\xc0d!\xc0",
+ out: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd",
+ outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd",
+ t: rotate,
+ }, {
+ desc: "misc correct and invalid, short src",
+ szDst: large,
+ atEOF: false,
+ in: "Hello\x80 w\x80orl\xc0d!\xc0",
+ out: "Hello\ufffd w\ufffdorl\ufffdd!",
+ outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd",
+ err: transform.ErrShortSrc,
+ t: rotate,
+ }, {
+ desc: "misc correct and invalid, short src, replacing RuneError",
+ szDst: large,
+ atEOF: false,
+ in: "Hel\ufffdlo\x80 w\x80orl\xc0d!\xc0",
+ out: "Hel?lo? w?orl?d!",
+ outFull: "Hel?lo? w?orl?d!?",
+ err: transform.ErrShortSrc,
+ t: Map(func(r rune) rune {
+ if r == utf8.RuneError {
+ return '?'
+ }
+ return r
+ }),
+ }} {
+ tt.check(t, i)
+ }
+}
+
+func TestRemove(t *testing.T) {
+ remove := Remove(Predicate(func(r rune) bool {
+ return strings.ContainsRune("aeiou\u0300\uFF24\U00012345", r)
+ }))
+
+ for i, tt := range []transformTest{
+ 0: {
+ szDst: large,
+ atEOF: true,
+ in: "",
+ out: "",
+ outFull: "",
+ t: remove,
+ },
+ 1: {
+ szDst: 0,
+ atEOF: true,
+ in: "aaaa",
+ out: "",
+ outFull: "",
+ t: remove,
+ },
+ 2: {
+ szDst: 1,
+ atEOF: true,
+ in: "aaaa",
+ out: "",
+ outFull: "",
+ t: remove,
+ },
+ 3: {
+ szDst: 1,
+ atEOF: true,
+ in: "baaaa",
+ out: "b",
+ outFull: "b",
+ t: remove,
+ },
+ 4: {
+ szDst: 2,
+ atEOF: true,
+ in: "açaaa",
+ out: "ç",
+ outFull: "ç",
+ t: remove,
+ },
+ 5: {
+ szDst: 2,
+ atEOF: true,
+ in: "aaaç",
+ out: "ç",
+ outFull: "ç",
+ t: remove,
+ },
+ 6: {
+ szDst: 2,
+ atEOF: false,
+ in: "a\x80",
+ out: "",
+ outFull: "\ufffd",
+ err: transform.ErrShortDst,
+ t: remove,
+ },
+ 7: {
+ szDst: 1,
+ atEOF: true,
+ in: "a\xc0",
+ out: "",
+ outFull: "\ufffd",
+ err: transform.ErrShortDst,
+ t: remove,
+ },
+ 8: {
+ szDst: 1,
+ atEOF: false,
+ in: "a\xc0",
+ out: "",
+ outFull: "\ufffd",
+ err: transform.ErrShortSrc,
+ t: remove,
+ },
+ 9: {
+ szDst: large,
+ atEOF: true,
+ in: "\x80",
+ out: "\ufffd",
+ outFull: "\ufffd",
+ t: remove,
+ },
+ 10: {
+ szDst: large,
+ atEOF: false,
+ in: "\x80",
+ out: "\ufffd",
+ outFull: "\ufffd",
+ t: remove,
+ },
+ 11: {
+ szDst: large,
+ atEOF: true,
+ in: "\xc0",
+ out: "\ufffd",
+ outFull: "\ufffd",
+ t: remove,
+ },
+ 12: {
+ szDst: large,
+ atEOF: true,
+ in: "Hello \U00012345world!",
+ out: "Hll wrld!",
+ outFull: "Hll wrld!",
+ t: remove,
+ },
+ 13: {
+ szDst: large,
+ atEOF: true,
+ in: "Hello\x80 w\x80orl\xc0d!\xc0",
+ out: "Hll\ufffd w\ufffdrl\ufffdd!\ufffd",
+ outFull: "Hll\ufffd w\ufffdrl\ufffdd!\ufffd",
+ t: remove,
+ },
+ 14: {
+ szDst: large,
+ atEOF: false,
+ in: "Hello\x80 w\x80orl\xc0d!\xc0",
+ out: "Hll\ufffd w\ufffdrl\ufffdd!",
+ outFull: "Hll\ufffd w\ufffdrl\ufffdd!\ufffd",
+ err: transform.ErrShortSrc,
+ t: remove,
+ },
+ 15: {
+ szDst: large,
+ atEOF: false,
+ in: "Hel\ufffdlo\x80 w\x80orl\xc0d!\xc0",
+ out: "Hello world!",
+ outFull: "Hello world!",
+ err: transform.ErrShortSrc,
+ t: Remove(Predicate(func(r rune) bool { return r == utf8.RuneError })),
+ },
+ 16: {
+ szDst: 4,
+ atEOF: true,
+ in: "Hellø",
+ out: "Hll",
+ outFull: "Hllø",
+ err: transform.ErrShortDst,
+ t: remove,
+ },
+ 17: {
+ szDst: 4,
+ atEOF: false,
+ in: "Hellø",
+ out: "Hll",
+ outFull: "Hllø",
+ err: transform.ErrShortDst,
+ t: remove,
+ },
+ 18: {
+ szDst: 8,
+ atEOF: false,
+ in: "\x80Hello\uFF24\x80",
+ out: "\ufffdHll",
+ outFull: "\ufffdHll\ufffd",
+ err: transform.ErrShortDst,
+ t: remove,
+ },
+ } {
+ tt.check(t, i)
+ }
+}
+
+func TestMapAlloc(t *testing.T) {
+ if n := testing.AllocsPerRun(3, func() {
+ Map(idem).Transform(nil, nil, false)
+ }); n > 0 {
+ t.Errorf("got %f; want 0", n)
+ }
+}
+
+func rmNop(r rune) bool { return false }
+
+func TestRemoveAlloc(t *testing.T) {
+ if n := testing.AllocsPerRun(3, func() {
+ Remove(Predicate(rmNop)).Transform(nil, nil, false)
+ }); n > 0 {
+ t.Errorf("got %f; want 0", n)
+ }
+}
+
+func BenchmarkRemove(b *testing.B) {
+ dst := make([]byte, len(input))
+ src := []byte(input)
+
+ r := Remove(Predicate(func(r rune) bool { return r == 'e' }))
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ r.Transform(dst, src, true)
+ }
+}
+
+func BenchmarkMapAll(b *testing.B) {
+ dst := make([]byte, 2*len(input))
+ src := []byte(input)
+
+ r := Map(func(r rune) rune { return 'a' })
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ r.Transform(dst, src, true)
+ }
+}
+
+func BenchmarkMapNone(b *testing.B) {
+ dst := make([]byte, 2*len(input))
+ src := []byte(input)
+
+ r := Map(func(r rune) rune { return r })
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ r.Transform(dst, src, true)
+ }
+}
+
+var (
+ input = strings.Repeat("Thé qüick brøwn føx jumps øver the lazy døg. ", 100)
+)
