blob: f6e8d8172510418f402297650afe8bd1104cdc39 [file] [log] [blame]
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package norm
import (
"strings"
"testing"
)
var iterBufSizes = []int{
MaxSegmentSize,
1.5 * MaxSegmentSize,
2 * MaxSegmentSize,
3 * MaxSegmentSize,
100 * MaxSegmentSize,
}
func doIterNorm(f Form, buf []byte, s string) []byte {
acc := []byte{}
i := Iter{}
i.SetInputString(f, s)
for !i.Done() {
n := i.Next(buf)
acc = append(acc, buf[:n]...)
}
return acc
}
func runIterTests(t *testing.T, name string, f Form, tests []AppendTest, norm bool) {
for i, test := range tests {
in := test.left + test.right
gold := test.out
if norm {
gold = string(f.AppendString(nil, test.out))
}
for _, sz := range iterBufSizes {
buf := make([]byte, sz)
out := string(doIterNorm(f, buf, in))
if len(out) != len(gold) {
const msg = "%s:%d:%d: length is %d; want %d"
t.Errorf(msg, name, i, sz, len(out), len(gold))
}
if out != gold {
// Find first rune that differs and show context.
ir := []rune(out)
ig := []rune(gold)
for j := 0; j < len(ir) && j < len(ig); j++ {
if ir[j] == ig[j] {
continue
}
if j -= 3; j < 0 {
j = 0
}
for e := j + 7; j < e && j < len(ir) && j < len(ig); j++ {
const msg = "%s:%d:%d: runeAt(%d) = %U; want %U"
t.Errorf(msg, name, i, sz, j, ir[j], ig[j])
}
break
}
}
}
}
}
func rep(r rune, n int) string {
return strings.Repeat(string(r), n)
}
var iterTests = []AppendTest{
{"", ascii, ascii},
{"", txt_all, txt_all},
{"", "a" + rep(0x0300, MaxSegmentSize/2), "a" + rep(0x0300, MaxSegmentSize/2)},
}
var iterTestsD = []AppendTest{
{ // segment overflow on unchanged character
"",
"a" + rep(0x0300, MaxSegmentSize/2) + "\u0316",
"a" + rep(0x0300, MaxSegmentSize/2-1) + "\u0316\u0300",
},
{ // segment overflow on unchanged character + start value
"",
"a" + rep(0x0300, MaxSegmentSize/2+maxCombiningChars+4) + "\u0316",
"a" + rep(0x0300, MaxSegmentSize/2+maxCombiningChars) + "\u0316" + rep(0x300, 4),
},
{ // segment overflow on decomposition
"",
"a" + rep(0x0300, MaxSegmentSize/2-1) + "\u0340",
"a" + rep(0x0300, MaxSegmentSize/2),
},
{ // segment overflow on decomposition + start value
"",
"a" + rep(0x0300, MaxSegmentSize/2-1) + "\u0340" + rep(0x300, maxCombiningChars+4) + "\u0320",
"a" + rep(0x0300, MaxSegmentSize/2-1) + rep(0x300, maxCombiningChars+1) + "\u0320" + rep(0x300, 4),
},
{ // start value after ASCII overflow
"",
rep('a', MaxSegmentSize) + rep(0x300, maxCombiningChars+2) + "\u0320",
rep('a', MaxSegmentSize) + rep(0x300, maxCombiningChars) + "\u0320\u0300\u0300",
},
{ // start value after Hangul overflow
"",
rep(0xAC00, MaxSegmentSize/6) + rep(0x300, maxCombiningChars+2) + "\u0320",
strings.Repeat("\u1100\u1161", MaxSegmentSize/6) + rep(0x300, maxCombiningChars-1) + "\u0320" + rep(0x300, 3),
},
{ // start value after cc=0
"",
"您您" + rep(0x300, maxCombiningChars+4) + "\u0320",
"您您" + rep(0x300, maxCombiningChars) + "\u0320" + rep(0x300, 4),
},
{ // start value after normalization
"",
"\u0300\u0320a" + rep(0x300, maxCombiningChars+4) + "\u0320",
"\u0320\u0300a" + rep(0x300, maxCombiningChars) + "\u0320" + rep(0x300, 4),
},
}
var iterTestsC = []AppendTest{
{ // ordering of non-composing combining characters
"",
"\u0305\u0316",
"\u0316\u0305",
},
{ // segment overflow
"",
"a" + rep(0x0305, MaxSegmentSize/2+4) + "\u0316",
"a" + rep(0x0305, MaxSegmentSize/2-1) + "\u0316" + rep(0x305, 5),
},
}
func TestIterNextD(t *testing.T) {
runIterTests(t, "IterNextD1", NFKD, appendTests, true)
runIterTests(t, "IterNextD2", NFKD, iterTests, true)
runIterTests(t, "IterNextD3", NFKD, iterTestsD, false)
}
func TestIterNextC(t *testing.T) {
runIterTests(t, "IterNextC1", NFKC, appendTests, true)
runIterTests(t, "IterNextC2", NFKC, iterTests, true)
runIterTests(t, "IterNextC3", NFKC, iterTestsC, false)
}
type SegmentTest struct {
in string
out []string
}
var segmentTests = []SegmentTest{
{rep('a', MaxSegmentSize), []string{rep('a', MaxSegmentSize), ""}},
{rep('a', MaxSegmentSize+2), []string{rep('a', MaxSegmentSize-1), "aaa", ""}},
{rep('a', MaxSegmentSize) + "\u0300aa", []string{rep('a', MaxSegmentSize-1), "a\u0300", "aa", ""}},
}
// Note that, by design, segmentation is equal for composing and decomposing forms.
func TestIterSegmentation(t *testing.T) {
segmentTest(t, "SegmentTestD", NFD, segmentTests)
segmentTest(t, "SegmentTestC", NFC, segmentTests)
}
func segmentTest(t *testing.T, name string, f Form, tests []SegmentTest) {
iter := Iter{}
for i, tt := range segmentTests {
buf := make([]byte, MaxSegmentSize)
iter.SetInputString(f, tt.in)
for j, seg := range tt.out {
if seg == "" {
if !iter.Done() {
n := iter.Next(buf)
res := string(buf[:n])
t.Errorf(`%s:%d:%d: expected Done()==true, found segment "%s"`, name, i, j, res)
}
continue
}
if iter.Done() {
t.Errorf("%s:%d:%d: Done()==true, want false", name, i, j)
}
n := iter.Next(buf)
seg = f.String(seg)
if res := string(buf[:n]); res != seg {
t.Errorf(`%s:%d:%d" segment was "%s" (%d); want "%s" (%d)`, name, i, j, res, len(res), seg, len(seg))
}
}
}
}