width/transform_test.go

// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package width

import (
	"bytes"
	"strings"
	"testing"

	"golang.org/x/text/internal/testtext"
	"golang.org/x/text/transform"
)

func foldRune(r rune) (folded rune, ok bool) {
	alt, ok := mapRunes[r]
	if ok && alt.e&tagNeedsFold != 0 {
		return alt.r, true
	}
	return r, false
}

func widenRune(r rune) (wide rune, ok bool) {
	alt, ok := mapRunes[r]
	if k := alt.e.kind(); k == EastAsianHalfwidth || k == EastAsianNarrow {
		return alt.r, true
	}
	return r, false
}

func narrowRune(r rune) (narrow rune, ok bool) {
	alt, ok := mapRunes[r]
	if k := alt.e.kind(); k == EastAsianFullwidth || k == EastAsianWide || k == EastAsianAmbiguous {
		return alt.r, true
	}
	return r, false
}

func TestFoldSingleRunes(t *testing.T) {
	for r := rune(0); r < 0x1FFFF; r++ {
		if loSurrogate <= r && r <= hiSurrogate {
			continue
		}
		x, _ := foldRune(r)
		want := string(x)
		got := Fold.String(string(r))
		if got != want {
			t.Errorf("Fold().String(%U) = %+q; want %+q", r, got, want)
		}
	}
}

type transformTest struct {
	desc    string
	src     string
	nBuf    int
	nDst    int
	atEOF   bool
	dst     string
	nSrc    int
	err     error
	nSpan   int
	errSpan error
}

func (tc *transformTest) doTest(t *testing.T, tr Transformer) {
	testtext.Run(t, tc.desc, func(t *testing.T) {
		b := make([]byte, tc.nBuf)
		nDst, nSrc, err := tr.Transform(b, []byte(tc.src), tc.atEOF)
		if got := string(b[:nDst]); got != tc.dst[:nDst] {
			t.Errorf("dst was %+q; want %+q", got, tc.dst)
		}
		if nDst != tc.nDst {
			t.Errorf("nDst was %d; want %d", nDst, tc.nDst)
		}
		if nSrc != tc.nSrc {
			t.Errorf("nSrc was %d; want %d", nSrc, tc.nSrc)
		}
		if err != tc.err {
			t.Errorf("error was %v; want %v", err, tc.err)
		}
		if got := tr.String(tc.src); got != tc.dst {
			t.Errorf("String(%q) = %q; want %q", tc.src, got, tc.dst)
		}
		n, err := tr.Span([]byte(tc.src), tc.atEOF)
		if n != tc.nSpan || err != tc.errSpan {
			t.Errorf("Span: got %d, %v; want %d, %v", n, err, tc.nSpan, tc.errSpan)
		}
	})
}

func TestFold(t *testing.T) {
	for _, tc := range []transformTest{{
		desc:    "empty",
		src:     "",
		nBuf:    10,
		dst:     "",
		nDst:    0,
		nSrc:    0,
		atEOF:   false,
		err:     nil,
		nSpan:   0,
		errSpan: nil,
	}, {
		desc:    "short source 1",
		src:     "a\xc2",
		nBuf:    10,
		dst:     "a\xc2",
		nDst:    1,
		nSrc:    1,
		atEOF:   false,
		err:     transform.ErrShortSrc,
		nSpan:   1,
		errSpan: transform.ErrShortSrc,
	}, {
		desc:    "short source 2",
		src:     "a\xe0\x80",
		nBuf:    10,
		dst:     "a\xe0\x80",
		nDst:    1,
		nSrc:    1,
		atEOF:   false,
		err:     transform.ErrShortSrc,
		nSpan:   1,
		errSpan: transform.ErrShortSrc,
	}, {
		desc:    "incomplete but terminated source 1",
		src:     "a\xc2",
		nBuf:    10,
		dst:     "a\xc2",
		nDst:    2,
		nSrc:    2,
		atEOF:   true,
		err:     nil,
		nSpan:   2,
		errSpan: nil,
	}, {
		desc:    "incomplete but terminated source 2",
		src:     "a\xe0\x80",
		nBuf:    10,
		dst:     "a\xe0\x80",
		nDst:    3,
		nSrc:    3,
		atEOF:   true,
		err:     nil,
		nSpan:   3,
		errSpan: nil,
	}, {
		desc:    "exact fit dst",
		src:     "a\uff01",
		nBuf:    2,
		dst:     "a!",
		nDst:    2,
		nSrc:    4,
		atEOF:   false,
		err:     nil,
		nSpan:   1,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "exact fit dst and src ascii",
		src:     "ab",
		nBuf:    2,
		dst:     "ab",
		nDst:    2,
		nSrc:    2,
		atEOF:   true,
		err:     nil,
		nSpan:   2,
		errSpan: nil,
	}, {
		desc:    "empty dst",
		src:     "\u0300",
		nBuf:    0,
		dst:     "\u0300",
		nDst:    0,
		nSrc:    0,
		atEOF:   true,
		err:     transform.ErrShortDst,
		nSpan:   2,
		errSpan: nil,
	}, {
		desc:    "empty dst ascii",
		src:     "a",
		nBuf:    0,
		dst:     "a",
		nDst:    0,
		nSrc:    0,
		atEOF:   true,
		err:     transform.ErrShortDst,
		nSpan:   1,
		errSpan: nil,
	}, {
		desc:    "short dst 1",
		src:     "a\uffe0", // ￠
		nBuf:    2,
		dst:     "a\u00a2", // ¢
		nDst:    1,
		nSrc:    1,
		atEOF:   false,
		err:     transform.ErrShortDst,
		nSpan:   1,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "short dst 2",
		src:     "不夠",
		nBuf:    3,
		dst:     "不夠",
		nDst:    3,
		nSrc:    3,
		atEOF:   true,
		err:     transform.ErrShortDst,
		nSpan:   6,
		errSpan: nil,
	}, {
		desc:    "short dst fast path",
		src:     "fast",
		nDst:    3,
		dst:     "fast",
		nBuf:    3,
		nSrc:    3,
		atEOF:   true,
		err:     transform.ErrShortDst,
		nSpan:   4,
		errSpan: nil,
	}, {
		desc:    "short dst larger buffer",
		src:     "\uff21" + strings.Repeat("0", 127) + "B",
		nBuf:    128,
		dst:     "A" + strings.Repeat("0", 127) + "B",
		nDst:    128,
		nSrc:    130,
		atEOF:   true,
		err:     transform.ErrShortDst,
		nSpan:   0,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "fast path alternation",
		src:     "fast路徑fast路徑",
		nBuf:    20,
		dst:     "fast路徑fast路徑",
		nDst:    20,
		nSrc:    20,
		atEOF:   true,
		err:     nil,
		nSpan:   20,
		errSpan: nil,
	}} {
		tc.doTest(t, Fold)
	}
}

func TestWidenSingleRunes(t *testing.T) {
	for r := rune(0); r < 0x1FFFF; r++ {
		if loSurrogate <= r && r <= hiSurrogate {
			continue
		}
		alt, _ := widenRune(r)
		want := string(alt)
		got := Widen.String(string(r))
		if got != want {
			t.Errorf("Widen().String(%U) = %+q; want %+q", r, got, want)
		}
	}
}

func TestWiden(t *testing.T) {
	for _, tc := range []transformTest{{
		desc:    "empty",
		src:     "",
		nBuf:    10,
		dst:     "",
		nDst:    0,
		nSrc:    0,
		atEOF:   false,
		err:     nil,
		nSpan:   0,
		errSpan: nil,
	}, {
		desc:    "short source 1",
		src:     "a\xc2",
		nBuf:    10,
		dst:     "ａ\xc2",
		nDst:    3,
		nSrc:    1,
		atEOF:   false,
		err:     transform.ErrShortSrc,
		nSpan:   0,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "short source 2",
		src:     "a\xe0\x80",
		nBuf:    10,
		dst:     "ａ\xe0\x80",
		nDst:    3,
		nSrc:    1,
		atEOF:   false,
		err:     transform.ErrShortSrc,
		nSpan:   0,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "incomplete but terminated source 1",
		src:     "a\xc2",
		nBuf:    10,
		dst:     "ａ\xc2",
		nDst:    4,
		nSrc:    2,
		atEOF:   true,
		err:     nil,
		nSpan:   0,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "incomplete but terminated source 2",
		src:     "a\xe0\x80",
		nBuf:    10,
		dst:     "ａ\xe0\x80",
		nDst:    5,
		nSrc:    3,
		atEOF:   true,
		err:     nil,
		nSpan:   0,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "short source 1 some span",
		src:     "ａ\xc2",
		nBuf:    10,
		dst:     "ａ\xc2",
		nDst:    3,
		nSrc:    3,
		atEOF:   false,
		err:     transform.ErrShortSrc,
		nSpan:   3,
		errSpan: transform.ErrShortSrc,
	}, {
		desc:    "short source 2 some span",
		src:     "ａ\xe0\x80",
		nBuf:    10,
		dst:     "ａ\xe0\x80",
		nDst:    3,
		nSrc:    3,
		atEOF:   false,
		err:     transform.ErrShortSrc,
		nSpan:   3,
		errSpan: transform.ErrShortSrc,
	}, {
		desc:    "incomplete but terminated source 1 some span",
		src:     "ａ\xc2",
		nBuf:    10,
		dst:     "ａ\xc2",
		nDst:    4,
		nSrc:    4,
		atEOF:   true,
		err:     nil,
		nSpan:   4,
		errSpan: nil,
	}, {
		desc:    "incomplete but terminated source 2 some span",
		src:     "ａ\xe0\x80",
		nBuf:    10,
		dst:     "ａ\xe0\x80",
		nDst:    5,
		nSrc:    5,
		atEOF:   true,
		err:     nil,
		nSpan:   5,
		errSpan: nil,
	}, {
		desc:    "exact fit dst",
		src:     "a!",
		nBuf:    6,
		dst:     "ａ\uff01",
		nDst:    6,
		nSrc:    2,
		atEOF:   false,
		err:     nil,
		nSpan:   0,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "empty dst",
		src:     "\u0300",
		nBuf:    0,
		dst:     "\u0300",
		nDst:    0,
		nSrc:    0,
		atEOF:   true,
		err:     transform.ErrShortDst,
		nSpan:   2,
		errSpan: nil,
	}, {
		desc:    "empty dst ascii",
		src:     "a",
		nBuf:    0,
		dst:     "ａ",
		nDst:    0,
		nSrc:    0,
		atEOF:   true,
		err:     transform.ErrShortDst,
		nSpan:   0,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "short dst 1",
		src:     "a\uffe0",
		nBuf:    4,
		dst:     "ａ\uffe0",
		nDst:    3,
		nSrc:    1,
		atEOF:   false,
		err:     transform.ErrShortDst,
		nSpan:   0,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "short dst 2",
		src:     "不夠",
		nBuf:    3,
		dst:     "不夠",
		nDst:    3,
		nSrc:    3,
		atEOF:   true,
		err:     transform.ErrShortDst,
		nSpan:   6,
		errSpan: nil,
	}, {
		desc:    "short dst ascii",
		src:     "ascii",
		nBuf:    3,
		dst:     "ａｓｃｉｉ", // U+ff41, ...
		nDst:    3,
		nSrc:    1,
		atEOF:   true,
		err:     transform.ErrShortDst,
		nSpan:   0,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "ambiguous",
		src:     "\uffe9",
		nBuf:    4,
		dst:     "\u2190",
		nDst:    3,
		nSrc:    3,
		atEOF:   false,
		err:     nil,
		nSpan:   0,
		errSpan: transform.ErrEndOfSpan,
	}} {
		tc.doTest(t, Widen)
	}
}

func TestNarrowSingleRunes(t *testing.T) {
	for r := rune(0); r < 0x1FFFF; r++ {
		if loSurrogate <= r && r <= hiSurrogate {
			continue
		}
		alt, _ := narrowRune(r)
		want := string(alt)
		got := Narrow.String(string(r))
		if got != want {
			t.Errorf("Narrow().String(%U) = %+q; want %+q", r, got, want)
		}
	}
}

func TestNarrow(t *testing.T) {
	for _, tc := range []transformTest{{
		desc:    "empty",
		src:     "",
		nBuf:    10,
		dst:     "",
		nDst:    0,
		nSrc:    0,
		atEOF:   false,
		err:     nil,
		nSpan:   0,
		errSpan: nil,
	}, {
		desc:    "short source 1",
		src:     "a\xc2",
		nBuf:    10,
		dst:     "a\xc2",
		nDst:    1,
		nSrc:    1,
		atEOF:   false,
		err:     transform.ErrShortSrc,
		nSpan:   1,
		errSpan: transform.ErrShortSrc,
	}, {
		desc:    "short source 2",
		src:     "ａ\xe0\x80",
		nBuf:    10,
		dst:     "a\xe0\x80",
		nDst:    1,
		nSrc:    3,
		atEOF:   false,
		err:     transform.ErrShortSrc,
		nSpan:   0,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "incomplete but terminated source 1",
		src:     "ａ\xc2",
		nBuf:    10,
		dst:     "a\xc2",
		nDst:    2,
		nSrc:    4,
		atEOF:   true,
		err:     nil,
		nSpan:   0,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "incomplete but terminated source 2",
		src:     "ａ\xe0\x80",
		nBuf:    10,
		dst:     "a\xe0\x80",
		nDst:    3,
		nSrc:    5,
		atEOF:   true,
		err:     nil,
		nSpan:   0,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "exact fit dst",
		src:     "ａ\uff01",
		nBuf:    2,
		dst:     "a!",
		nDst:    2,
		nSrc:    6,
		atEOF:   false,
		err:     nil,
		nSpan:   0,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "exact fit dst some span",
		src:     "a\uff01",
		nBuf:    2,
		dst:     "a!",
		nDst:    2,
		nSrc:    4,
		atEOF:   false,
		err:     nil,
		nSpan:   1,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "empty dst",
		src:     "\u0300",
		nBuf:    0,
		dst:     "\u0300",
		nDst:    0,
		nSrc:    0,
		atEOF:   true,
		err:     transform.ErrShortDst,
		nSpan:   2,
		errSpan: nil,
	}, {
		desc:    "empty dst ascii",
		src:     "a",
		nBuf:    0,
		dst:     "a",
		nDst:    0,
		nSrc:    0,
		atEOF:   true,
		err:     transform.ErrShortDst,
		nSpan:   1,
		errSpan: nil,
	}, {
		desc:    "short dst 1",
		src:     "ａ\uffe0", // ￠
		nBuf:    2,
		dst:     "a\u00a2", // ¢
		nDst:    1,
		nSrc:    3,
		atEOF:   false,
		err:     transform.ErrShortDst,
		nSpan:   0,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "short dst 2",
		src:     "不夠",
		nBuf:    3,
		dst:     "不夠",
		nDst:    3,
		nSrc:    3,
		atEOF:   true,
		err:     transform.ErrShortDst,
		nSpan:   6,
		errSpan: nil,
	}, {
		// Create a narrow variant of ambiguous runes, if they exist.
		desc:    "ambiguous",
		src:     "\u2190",
		nBuf:    4,
		dst:     "\uffe9",
		nDst:    3,
		nSrc:    3,
		atEOF:   false,
		err:     nil,
		nSpan:   0,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "short dst fast path",
		src:     "fast",
		nBuf:    3,
		dst:     "fast",
		nDst:    3,
		nSrc:    3,
		atEOF:   true,
		err:     transform.ErrShortDst,
		nSpan:   4,
		errSpan: nil,
	}, {
		desc:    "short dst larger buffer",
		src:     "\uff21" + strings.Repeat("0", 127) + "B",
		nBuf:    128,
		dst:     "A" + strings.Repeat("0", 127) + "B",
		nDst:    128,
		nSrc:    130,
		atEOF:   true,
		err:     transform.ErrShortDst,
		nSpan:   0,
		errSpan: transform.ErrEndOfSpan,
	}, {
		desc:    "fast path alternation",
		src:     "fast路徑fast路徑",
		nBuf:    20,
		dst:     "fast路徑fast路徑",
		nDst:    20,
		nSrc:    20,
		atEOF:   true,
		err:     nil,
		nSpan:   20,
		errSpan: nil,
	}} {
		tc.doTest(t, Narrow)
	}
}

func bench(b *testing.B, t Transformer, s string) {
	dst := make([]byte, 1024)
	src := []byte(s)
	b.SetBytes(int64(len(src)))
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		t.Transform(dst, src, true)
	}
}

func changingRunes(f func(r rune) (rune, bool)) string {
	buf := &bytes.Buffer{}
	for r := rune(0); r <= 0xFFFF; r++ {
		if _, ok := foldRune(r); ok {
			buf.WriteRune(r)
		}
	}
	return buf.String()
}

func BenchmarkFoldASCII(b *testing.B) {
	bench(b, Fold, testtext.ASCII)
}

func BenchmarkFoldCJK(b *testing.B) {
	bench(b, Fold, testtext.CJK)
}

func BenchmarkFoldNonCanonical(b *testing.B) {
	bench(b, Fold, changingRunes(foldRune))
}

func BenchmarkFoldOther(b *testing.B) {
	bench(b, Fold, testtext.TwoByteUTF8+testtext.ThreeByteUTF8)
}

func BenchmarkWideASCII(b *testing.B) {
	bench(b, Widen, testtext.ASCII)
}

func BenchmarkWideCJK(b *testing.B) {
	bench(b, Widen, testtext.CJK)
}

func BenchmarkWideNonCanonical(b *testing.B) {
	bench(b, Widen, changingRunes(widenRune))
}

func BenchmarkWideOther(b *testing.B) {
	bench(b, Widen, testtext.TwoByteUTF8+testtext.ThreeByteUTF8)
}

func BenchmarkNarrowASCII(b *testing.B) {
	bench(b, Narrow, testtext.ASCII)
}

func BenchmarkNarrowCJK(b *testing.B) {
	bench(b, Narrow, testtext.CJK)
}

func BenchmarkNarrowNonCanonical(b *testing.B) {
	bench(b, Narrow, changingRunes(narrowRune))
}

func BenchmarkNarrowOther(b *testing.B) {
	bench(b, Narrow, testtext.TwoByteUTF8+testtext.ThreeByteUTF8)
}