src/index/suffixarray/suffixarray_test.go - go - Git at Google

 // Copyright 2010 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 package suffixarray

 import (
 	"bytes"
 	"fmt"
 	"io/fs"
 	"math/rand"
 	"os"
 	"path/filepath"
 	"regexp"
 	"sort"
 	"strings"
 	"testing"
 )

 type testCase struct {
 	name     string   // name of test case
 	source   string   // source to index
 	patterns []string // patterns to lookup
 }

 var testCases = []testCase{
 	{
 		"empty string",
 		"",
 		[]string{
 			"",
 			"foo",
 			"(foo)",
 			".*",
 			"a*",
 		},
 	},

 	{
 		"all a's",
 		"aaaaaaaaaa", // 10 a's
 		[]string{
 			"",
 			"a",
 			"aa",
 			"aaa",
 			"aaaa",
 			"aaaaa",
 			"aaaaaa",
 			"aaaaaaa",
 			"aaaaaaaa",
 			"aaaaaaaaa",
 			"aaaaaaaaaa",
 			"aaaaaaaaaaa", // 11 a's
 			".",
 			".*",
 			"a+",
 			"aa+",
 			"aaaa[b]?",
 			"aaa*",
 		},
 	},

 	{
 		"abc",
 		"abc",
 		[]string{
 			"a",
 			"b",
 			"c",
 			"ab",
 			"bc",
 			"abc",
 			"a.c",
 			"a(b|c)",
 			"abc?",
 		},
 	},

 	{
 		"barbara*3",
 		"barbarabarbarabarbara",
 		[]string{
 			"a",
 			"bar",
 			"rab",
 			"arab",
 			"barbar",
 			"bara?bar",
 		},
 	},

 	{
 		"typing drill",
 		"Now is the time for all good men to come to the aid of their country.",
 		[]string{
 			"Now",
 			"the time",
 			"to come the aid",
 			"is the time for all good men to come to the aid of their",
 			"to (come|the)?",
 		},
 	},

 	{
 		"godoc simulation",
 		"package main\n\nimport(\n    \"rand\"\n    ",
 		[]string{},
 	},
 }

 // find all occurrences of s in source; report at most n occurrences
 func find(src, s string, n int) []int {
 	var res []int
 	if s != "" && n != 0 {
 		// find at most n occurrences of s in src
 		for i := -1; n < 0 || len(res) < n; {
 			j := strings.Index(src[i+1:], s)
 			if j < 0 {
 				break
 			}
 			i += j + 1
 			res = append(res, i)
 		}
 	}
 	return res
 }

 func testLookup(t *testing.T, tc *testCase, x *Index, s string, n int) {
 	res := x.Lookup([]byte(s), n)
 	exp := find(tc.source, s, n)

 	// check that the lengths match
 	if len(res) != len(exp) {
 		t.Errorf("test %q, lookup %q (n = %d): expected %d results; got %d", tc.name, s, n, len(exp), len(res))
 	}

 	// if n >= 0 the number of results is limited --- unless n >= all results,
 	// we may obtain different positions from the Index and from find (because
 	// Index may not find the results in the same order as find) => in general
 	// we cannot simply check that the res and exp lists are equal

 	// check that each result is in fact a correct match and there are no duplicates
 	sort.Ints(res)
 	for i, r := range res {
 		if r < 0 || len(tc.source) <= r {
 			t.Errorf("test %q, lookup %q, result %d (n = %d): index %d out of range [0, %d[", tc.name, s, i, n, r, len(tc.source))
 		} else if !strings.HasPrefix(tc.source[r:], s) {
 			t.Errorf("test %q, lookup %q, result %d (n = %d): index %d not a match", tc.name, s, i, n, r)
 		}
 		if i > 0 && res[i-1] == r {
 			t.Errorf("test %q, lookup %q, result %d (n = %d): found duplicate index %d", tc.name, s, i, n, r)
 		}
 	}

 	if n < 0 {
 		// all results computed - sorted res and exp must be equal
 		for i, r := range res {
 			e := exp[i]
 			if r != e {
 				t.Errorf("test %q, lookup %q, result %d: expected index %d; got %d", tc.name, s, i, e, r)
 			}
 		}
 	}
 }

 func testFindAllIndex(t *testing.T, tc *testCase, x *Index, rx *regexp.Regexp, n int) {
 	res := x.FindAllIndex(rx, n)
 	exp := rx.FindAllStringIndex(tc.source, n)

 	// check that the lengths match
 	if len(res) != len(exp) {
 		t.Errorf("test %q, FindAllIndex %q (n = %d): expected %d results; got %d", tc.name, rx, n, len(exp), len(res))
 	}

 	// if n >= 0 the number of results is limited --- unless n >= all results,
 	// we may obtain different positions from the Index and from regexp (because
 	// Index may not find the results in the same order as regexp) => in general
 	// we cannot simply check that the res and exp lists are equal

 	// check that each result is in fact a correct match and the result is sorted
 	for i, r := range res {
 		if r[0] < 0 || r[0] > r[1] || len(tc.source) < r[1] {
 			t.Errorf("test %q, FindAllIndex %q, result %d (n == %d): illegal match [%d, %d]", tc.name, rx, i, n, r[0], r[1])
 		} else if !rx.MatchString(tc.source[r[0]:r[1]]) {
 			t.Errorf("test %q, FindAllIndex %q, result %d (n = %d): [%d, %d] not a match", tc.name, rx, i, n, r[0], r[1])
 		}
 	}

 	if n < 0 {
 		// all results computed - sorted res and exp must be equal
 		for i, r := range res {
 			e := exp[i]
 			if r[0] != e[0] || r[1] != e[1] {
 				t.Errorf("test %q, FindAllIndex %q, result %d: expected match [%d, %d]; got [%d, %d]",
 					tc.name, rx, i, e[0], e[1], r[0], r[1])
 			}
 		}
 	}
 }

 func testLookups(t *testing.T, tc *testCase, x *Index, n int) {
 	for _, pat := range tc.patterns {
 		testLookup(t, tc, x, pat, n)
 		if rx, err := regexp.Compile(pat); err == nil {
 			testFindAllIndex(t, tc, x, rx, n)
 		}
 	}
 }

 // index is used to hide the sort.Interface
 type index Index

 func (x *index) Len() int           { return x.sa.len() }
 func (x *index) Less(i, j int) bool { return bytes.Compare(x.at(i), x.at(j)) < 0 }
 func (x *index) Swap(i, j int) {
 	if x.sa.int32 != nil {
 		x.sa.int32[i], x.sa.int32[j] = x.sa.int32[j], x.sa.int32[i]
 	} else {
 		x.sa.int64[i], x.sa.int64[j] = x.sa.int64[j], x.sa.int64[i]
 	}
 }

 func (x *index) at(i int) []byte {
 	return x.data[x.sa.get(i):]
 }

 func testConstruction(t *testing.T, tc *testCase, x *Index) {
 	if !sort.IsSorted((*index)(x)) {
 		t.Errorf("failed testConstruction %s", tc.name)
 	}
 }

 func equal(x, y *Index) bool {
 	if !bytes.Equal(x.data, y.data) {
 		return false
 	}
 	if x.sa.len() != y.sa.len() {
 		return false
 	}
 	n := x.sa.len()
 	for i := 0; i < n; i++ {
 		if x.sa.get(i) != y.sa.get(i) {
 			return false
 		}
 	}
 	return true
 }

 // returns the serialized index size
 func testSaveRestore(t *testing.T, tc *testCase, x *Index) int {
 	var buf bytes.Buffer
 	if err := x.Write(&buf); err != nil {
 		t.Errorf("failed writing index %s (%s)", tc.name, err)
 	}
 	size := buf.Len()
 	var y Index
 	if err := y.Read(bytes.NewReader(buf.Bytes())); err != nil {
 		t.Errorf("failed reading index %s (%s)", tc.name, err)
 	}
 	if !equal(x, &y) {
 		t.Errorf("restored index doesn't match saved index %s", tc.name)
 	}

 	old := maxData32
 	defer func() {
 		maxData32 = old
 	}()
 	// Reread as forced 32.
 	y = Index{}
 	maxData32 = realMaxData32
 	if err := y.Read(bytes.NewReader(buf.Bytes())); err != nil {
 		t.Errorf("failed reading index %s (%s)", tc.name, err)
 	}
 	if !equal(x, &y) {
 		t.Errorf("restored index doesn't match saved index %s", tc.name)
 	}

 	// Reread as forced 64.
 	y = Index{}
 	maxData32 = -1
 	if err := y.Read(bytes.NewReader(buf.Bytes())); err != nil {
 		t.Errorf("failed reading index %s (%s)", tc.name, err)
 	}
 	if !equal(x, &y) {
 		t.Errorf("restored index doesn't match saved index %s", tc.name)
 	}

 	return size
 }

 func testIndex(t *testing.T) {
 	for _, tc := range testCases {
 		x := New([]byte(tc.source))
 		testConstruction(t, &tc, x)
 		testSaveRestore(t, &tc, x)
 		testLookups(t, &tc, x, 0)
 		testLookups(t, &tc, x, 1)
 		testLookups(t, &tc, x, 10)
 		testLookups(t, &tc, x, 2e9)
 		testLookups(t, &tc, x, -1)
 	}
 }

 func TestIndex32(t *testing.T) {
 	testIndex(t)
 }

 func TestIndex64(t *testing.T) {
 	maxData32 = -1
 	defer func() {
 		maxData32 = realMaxData32
 	}()
 	testIndex(t)
 }

 func TestNew32(t *testing.T) {
 	test(t, func(x []byte) []int {
 		sa := make([]int32, len(x))
 		text_32(x, sa)
 		out := make([]int, len(sa))
 		for i, v := range sa {
 			out[i] = int(v)
 		}
 		return out
 	})
 }

 func TestNew64(t *testing.T) {
 	test(t, func(x []byte) []int {
 		sa := make([]int64, len(x))
 		text_64(x, sa)
 		out := make([]int, len(sa))
 		for i, v := range sa {
 			out[i] = int(v)
 		}
 		return out
 	})
 }

 // test tests an arbitrary suffix array construction function.
 // Generates many inputs, builds and checks suffix arrays.
 func test(t *testing.T, build func([]byte) []int) {
 	t.Run("ababab...", func(t *testing.T) {
 		// Very repetitive input has numLMS = len(x)/2-1
 		// at top level, the largest it can be.
 		// But maxID is only two (aba and ab$).
 		size := 100000
 		if testing.Short() {
 			size = 10000
 		}
 		x := make([]byte, size)
 		for i := range x {
 			x[i] = "ab"[i%2]
 		}
 		testSA(t, x, build)
 	})

 	t.Run("forcealloc", func(t *testing.T) {
 		// Construct a pathological input that forces
 		// recurse_32 to allocate a new temporary buffer.
 		// The input must have more than N/3 LMS-substrings,
 		// which we arrange by repeating an SLSLSLSLSLSL pattern
 		// like ababab... above, but then we must also arrange
 		// for a large number of distinct LMS-substrings.
 		// We use this pattern:
 		// 1 255 1 254 1 253 1 ... 1 2 1 255 2 254 2 253 2 252 2 ...
 		// This gives approximately 2¹⁵ distinct LMS-substrings.
 		// We need to repeat at least one substring, though,
 		// or else the recursion can be bypassed entirely.
 		x := make([]byte, 100000, 100001)
 		lo := byte(1)
 		hi := byte(255)
 		for i := range x {
 			if i%2 == 0 {
 				x[i] = lo
 			} else {
 				x[i] = hi
 				hi--
 				if hi <= lo {
 					lo++
 					if lo == 0 {
 						lo = 1
 					}
 					hi = 255
 				}
 			}
 		}
 		x[:cap(x)][len(x)] = 0 // for sais.New
 		testSA(t, x, build)
 	})

 	t.Run("exhaustive2", func(t *testing.T) {
 		// All inputs over {0,1} up to length 21.
 		// Runs in about 10 seconds on my laptop.
 		x := make([]byte, 30)
 		numFail := 0
 		for n := 0; n <= 21; n++ {
 			if n > 12 && testing.Short() {
 				break
 			}
 			x[n] = 0 // for sais.New
 			testRec(t, x[:n], 0, 2, &numFail, build)
 		}
 	})

 	t.Run("exhaustive3", func(t *testing.T) {
 		// All inputs over {0,1,2} up to length 14.
 		// Runs in about 10 seconds on my laptop.
 		x := make([]byte, 30)
 		numFail := 0
 		for n := 0; n <= 14; n++ {
 			if n > 8 && testing.Short() {
 				break
 			}
 			x[n] = 0 // for sais.New
 			testRec(t, x[:n], 0, 3, &numFail, build)
 		}
 	})
 }

 // testRec fills x[i:] with all possible combinations of values in [1,max]
 // and then calls testSA(t, x, build) for each one.
 func testRec(t *testing.T, x []byte, i, max int, numFail *int, build func([]byte) []int) {
 	if i < len(x) {
 		for x[i] = 1; x[i] <= byte(max); x[i]++ {
 			testRec(t, x, i+1, max, numFail, build)
 		}
 		return
 	}

 	if !testSA(t, x, build) {
 		*numFail++
 		if *numFail >= 10 {
 			t.Errorf("stopping after %d failures", *numFail)
 			t.FailNow()
 		}
 	}
 }

 // testSA tests the suffix array build function on the input x.
 // It constructs the suffix array and then checks that it is correct.
 func testSA(t *testing.T, x []byte, build func([]byte) []int) bool {
 	defer func() {
 		if e := recover(); e != nil {
 			t.Logf("build %v", x)
 			panic(e)
 		}
 	}()
 	sa := build(x)
 	if len(sa) != len(x) {
 		t.Errorf("build %v: len(sa) = %d, want %d", x, len(sa), len(x))
 		return false
 	}
 	for i := 0; i+1 < len(sa); i++ {
 		if sa[i] < 0 || sa[i] >= len(x) || sa[i+1] < 0 || sa[i+1] >= len(x) {
 			t.Errorf("build %s: sa out of range: %v\n", x, sa)
 			return false
 		}
 		if bytes.Compare(x[sa[i]:], x[sa[i+1]:]) >= 0 {
 			t.Errorf("build %v -> %v\nsa[%d:] = %d,%d out of order", x, sa, i, sa[i], sa[i+1])
 			return false
 		}
 	}

 	return true
 }

 var (
 	benchdata = make([]byte, 1e6)
 	benchrand = make([]byte, 1e6)
 )

 // Of all possible inputs, the random bytes have the least amount of substring
 // repetition, and the repeated bytes have the most. For most algorithms,
 // the running time of every input will be between these two.
 func benchmarkNew(b *testing.B, random bool) {
 	b.ReportAllocs()
 	b.StopTimer()
 	data := benchdata
 	if random {
 		data = benchrand
 		if data[0] == 0 {
 			for i := range data {
 				data[i] = byte(rand.Intn(256))
 			}
 		}
 	}
 	b.StartTimer()
 	b.SetBytes(int64(len(data)))
 	for i := 0; i < b.N; i++ {
 		New(data)
 	}
 }

 func makeText(name string) ([]byte, error) {
 	var data []byte
 	switch name {
 	case "opticks":
 		var err error
 		data, err = os.ReadFile("../../testdata/Isaac.Newton-Opticks.txt")
 		if err != nil {
 			return nil, err
 		}
 	case "go":
 		err := filepath.WalkDir("../..", func(path string, info fs.DirEntry, err error) error {
 			if err == nil && strings.HasSuffix(path, ".go") && !info.IsDir() {
 				file, err := os.ReadFile(path)
 				if err != nil {
 					return err
 				}
 				data = append(data, file...)
 			}
 			return nil
 		})
 		if err != nil {
 			return nil, err
 		}
 	case "zero":
 		data = make([]byte, 50e6)
 	case "rand":
 		data = make([]byte, 50e6)
 		for i := range data {
 			data[i] = byte(rand.Intn(256))
 		}
 	}
 	return data, nil
 }

 func setBits(bits int) (cleanup func()) {
 	if bits == 32 {
 		maxData32 = realMaxData32
 	} else {
 		maxData32 = -1 // force use of 64-bit code
 	}
 	return func() {
 		maxData32 = realMaxData32
 	}
 }

 func BenchmarkNew(b *testing.B) {
 	for _, text := range []string{"opticks", "go", "zero", "rand"} {
 		b.Run("text="+text, func(b *testing.B) {
 			data, err := makeText(text)
 			if err != nil {
 				b.Fatal(err)
 			}
 			if testing.Short() && len(data) > 5e6 {
 				data = data[:5e6]
 			}
 			for _, size := range []int{100e3, 500e3, 1e6, 5e6, 10e6, 50e6} {
 				if len(data) < size {
 					continue
 				}
 				data := data[:size]
 				name := fmt.Sprintf("%dK", size/1e3)
 				if size >= 1e6 {
 					name = fmt.Sprintf("%dM", size/1e6)
 				}
 				b.Run("size="+name, func(b *testing.B) {
 					for _, bits := range []int{32, 64} {
 						if ^uint(0) == 0xffffffff && bits == 64 {
 							continue
 						}
 						b.Run(fmt.Sprintf("bits=%d", bits), func(b *testing.B) {
 							cleanup := setBits(bits)
 							defer cleanup()

 							b.SetBytes(int64(len(data)))
 							b.ReportAllocs()
 							for i := 0; i < b.N; i++ {
 								New(data)
 							}
 						})
 					}
 				})
 			}
 		})
 	}
 }

 func BenchmarkSaveRestore(b *testing.B) {
 	r := rand.New(rand.NewSource(0x5a77a1)) // guarantee always same sequence
 	data := make([]byte, 1<<20)             // 1MB of data to index
 	for i := range data {
 		data[i] = byte(r.Intn(256))
 	}
 	for _, bits := range []int{32, 64} {
 		if ^uint(0) == 0xffffffff && bits == 64 {
 			continue
 		}
 		b.Run(fmt.Sprintf("bits=%d", bits), func(b *testing.B) {
 			cleanup := setBits(bits)
 			defer cleanup()

 			b.StopTimer()
 			x := New(data)
 			size := testSaveRestore(nil, nil, x)       // verify correctness
 			buf := bytes.NewBuffer(make([]byte, size)) // avoid growing
 			b.SetBytes(int64(size))
 			b.StartTimer()
 			b.ReportAllocs()
 			for i := 0; i < b.N; i++ {
 				buf.Reset()
 				if err := x.Write(buf); err != nil {
 					b.Fatal(err)
 				}
 				var y Index
 				if err := y.Read(buf); err != nil {
 					b.Fatal(err)
 				}
 			}
 		})
 	}
 }
	// Copyright 2010 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	package suffixarray

	import (
	"bytes"
	"fmt"
	"io/fs"
	"math/rand"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"testing"
	)

	type testCase struct {
	name string // name of test case
	source string // source to index
	patterns []string // patterns to lookup
	}

	var testCases = []testCase{
	{
	"empty string",
	"",
	[]string{
	"",
	"foo",
	"(foo)",
	".*",
	"a*",
	},
	},

	{
	"all a's",
	"aaaaaaaaaa", // 10 a's
	[]string{
	"",
	"a",
	"aa",
	"aaa",
	"aaaa",
	"aaaaa",
	"aaaaaa",
	"aaaaaaa",
	"aaaaaaaa",
	"aaaaaaaaa",
	"aaaaaaaaaa",
	"aaaaaaaaaaa", // 11 a's
	".",
	".*",
	"a+",
	"aa+",
	"aaaa[b]?",
	"aaa*",
	},
	},

	{
	"abc",
	"abc",
	[]string{
	"a",
	"b",
	"c",
	"ab",
	"bc",
	"abc",
	"a.c",
	"a(b\|c)",
	"abc?",
	},
	},

	{
	"barbara*3",
	"barbarabarbarabarbara",
	[]string{
	"a",
	"bar",
	"rab",
	"arab",
	"barbar",
	"bara?bar",
	},
	},

	{
	"typing drill",
	"Now is the time for all good men to come to the aid of their country.",
	[]string{
	"Now",
	"the time",
	"to come the aid",
	"is the time for all good men to come to the aid of their",
	"to (come\|the)?",
	},
	},

	{
	"godoc simulation",
	"package main\n\nimport(\n \"rand\"\n ",
	[]string{},
	},
	}

	// find all occurrences of s in source; report at most n occurrences
	func find(src, s string, n int) []int {
	var res []int
	if s != "" && n != 0 {
	// find at most n occurrences of s in src
	for i := -1; n < 0 \|\| len(res) < n; {
	j := strings.Index(src[i+1:], s)
	if j < 0 {
	break
	}
	i += j + 1
	res = append(res, i)
	}
	}
	return res
	}

	func testLookup(t testing.T, tc testCase, x *Index, s string, n int) {
	res := x.Lookup([]byte(s), n)
	exp := find(tc.source, s, n)

	// check that the lengths match
	if len(res) != len(exp) {
	t.Errorf("test %q, lookup %q (n = %d): expected %d results; got %d", tc.name, s, n, len(exp), len(res))
	}

	// if n >= 0 the number of results is limited --- unless n >= all results,
	// we may obtain different positions from the Index and from find (because
	// Index may not find the results in the same order as find) => in general
	// we cannot simply check that the res and exp lists are equal

	// check that each result is in fact a correct match and there are no duplicates
	sort.Ints(res)
	for i, r := range res {
	if r < 0 \|\| len(tc.source) <= r {
	t.Errorf("test %q, lookup %q, result %d (n = %d): index %d out of range [0, %d[", tc.name, s, i, n, r, len(tc.source))
	} else if !strings.HasPrefix(tc.source[r:], s) {
	t.Errorf("test %q, lookup %q, result %d (n = %d): index %d not a match", tc.name, s, i, n, r)
	}
	if i > 0 && res[i-1] == r {
	t.Errorf("test %q, lookup %q, result %d (n = %d): found duplicate index %d", tc.name, s, i, n, r)
	}
	}

	if n < 0 {
	// all results computed - sorted res and exp must be equal
	for i, r := range res {
	e := exp[i]
	if r != e {
	t.Errorf("test %q, lookup %q, result %d: expected index %d; got %d", tc.name, s, i, e, r)
	}
	}
	}
	}

	func testFindAllIndex(t testing.T, tc testCase, x Index, rx regexp.Regexp, n int) {
	res := x.FindAllIndex(rx, n)
	exp := rx.FindAllStringIndex(tc.source, n)

	// check that the lengths match
	if len(res) != len(exp) {
	t.Errorf("test %q, FindAllIndex %q (n = %d): expected %d results; got %d", tc.name, rx, n, len(exp), len(res))
	}

	// if n >= 0 the number of results is limited --- unless n >= all results,
	// we may obtain different positions from the Index and from regexp (because
	// Index may not find the results in the same order as regexp) => in general
	// we cannot simply check that the res and exp lists are equal

	// check that each result is in fact a correct match and the result is sorted
	for i, r := range res {
	if r[0] < 0 \|\| r[0] > r[1] \|\| len(tc.source) < r[1] {
	t.Errorf("test %q, FindAllIndex %q, result %d (n == %d): illegal match [%d, %d]", tc.name, rx, i, n, r[0], r[1])
	} else if !rx.MatchString(tc.source[r[0]:r[1]]) {
	t.Errorf("test %q, FindAllIndex %q, result %d (n = %d): [%d, %d] not a match", tc.name, rx, i, n, r[0], r[1])
	}
	}

	if n < 0 {
	// all results computed - sorted res and exp must be equal
	for i, r := range res {
	e := exp[i]
	if r[0] != e[0] \|\| r[1] != e[1] {
	t.Errorf("test %q, FindAllIndex %q, result %d: expected match [%d, %d]; got [%d, %d]",
	tc.name, rx, i, e[0], e[1], r[0], r[1])
	}
	}
	}
	}

	func testLookups(t testing.T, tc testCase, x *Index, n int) {
	for _, pat := range tc.patterns {
	testLookup(t, tc, x, pat, n)
	if rx, err := regexp.Compile(pat); err == nil {
	testFindAllIndex(t, tc, x, rx, n)
	}
	}
	}

	// index is used to hide the sort.Interface
	type index Index

	func (x *index) Len() int { return x.sa.len() }
	func (x *index) Less(i, j int) bool { return bytes.Compare(x.at(i), x.at(j)) < 0 }
	func (x *index) Swap(i, j int) {
	if x.sa.int32 != nil {
	x.sa.int32[i], x.sa.int32[j] = x.sa.int32[j], x.sa.int32[i]
	} else {
	x.sa.int64[i], x.sa.int64[j] = x.sa.int64[j], x.sa.int64[i]
	}
	}

	func (x *index) at(i int) []byte {
	return x.data[x.sa.get(i):]
	}

	func testConstruction(t testing.T, tc testCase, x *Index) {
	if !sort.IsSorted((*index)(x)) {
	t.Errorf("failed testConstruction %s", tc.name)
	}
	}

	func equal(x, y *Index) bool {
	if !bytes.Equal(x.data, y.data) {
	return false
	}
	if x.sa.len() != y.sa.len() {
	return false
	}
	n := x.sa.len()
	for i := 0; i < n; i++ {
	if x.sa.get(i) != y.sa.get(i) {
	return false
	}
	}
	return true
	}

	// returns the serialized index size
	func testSaveRestore(t testing.T, tc testCase, x *Index) int {
	var buf bytes.Buffer
	if err := x.Write(&buf); err != nil {
	t.Errorf("failed writing index %s (%s)", tc.name, err)
	}
	size := buf.Len()
	var y Index
	if err := y.Read(bytes.NewReader(buf.Bytes())); err != nil {
	t.Errorf("failed reading index %s (%s)", tc.name, err)
	}
	if !equal(x, &y) {
	t.Errorf("restored index doesn't match saved index %s", tc.name)
	}

	old := maxData32
	defer func() {
	maxData32 = old
	}()
	// Reread as forced 32.
	y = Index{}
	maxData32 = realMaxData32
	if err := y.Read(bytes.NewReader(buf.Bytes())); err != nil {
	t.Errorf("failed reading index %s (%s)", tc.name, err)
	}
	if !equal(x, &y) {
	t.Errorf("restored index doesn't match saved index %s", tc.name)
	}

	// Reread as forced 64.
	y = Index{}
	maxData32 = -1
	if err := y.Read(bytes.NewReader(buf.Bytes())); err != nil {
	t.Errorf("failed reading index %s (%s)", tc.name, err)
	}
	if !equal(x, &y) {
	t.Errorf("restored index doesn't match saved index %s", tc.name)
	}

	return size
	}

	func testIndex(t *testing.T) {
	for _, tc := range testCases {
	x := New([]byte(tc.source))
	testConstruction(t, &tc, x)
	testSaveRestore(t, &tc, x)
	testLookups(t, &tc, x, 0)
	testLookups(t, &tc, x, 1)
	testLookups(t, &tc, x, 10)
	testLookups(t, &tc, x, 2e9)
	testLookups(t, &tc, x, -1)
	}
	}

	func TestIndex32(t *testing.T) {
	testIndex(t)
	}

	func TestIndex64(t *testing.T) {
	maxData32 = -1
	defer func() {
	maxData32 = realMaxData32
	}()
	testIndex(t)
	}

	func TestNew32(t *testing.T) {
	test(t, func(x []byte) []int {
	sa := make([]int32, len(x))
	text_32(x, sa)
	out := make([]int, len(sa))
	for i, v := range sa {
	out[i] = int(v)
	}
	return out
	})
	}

	func TestNew64(t *testing.T) {
	test(t, func(x []byte) []int {
	sa := make([]int64, len(x))
	text_64(x, sa)
	out := make([]int, len(sa))
	for i, v := range sa {
	out[i] = int(v)
	}
	return out
	})
	}

	// test tests an arbitrary suffix array construction function.
	// Generates many inputs, builds and checks suffix arrays.
	func test(t *testing.T, build func([]byte) []int) {
	t.Run("ababab...", func(t *testing.T) {
	// Very repetitive input has numLMS = len(x)/2-1
	// at top level, the largest it can be.
	// But maxID is only two (aba and ab$).
	size := 100000
	if testing.Short() {
	size = 10000
	}
	x := make([]byte, size)
	for i := range x {
	x[i] = "ab"[i%2]
	}
	testSA(t, x, build)
	})

	t.Run("forcealloc", func(t *testing.T) {
	// Construct a pathological input that forces
	// recurse_32 to allocate a new temporary buffer.
	// The input must have more than N/3 LMS-substrings,
	// which we arrange by repeating an SLSLSLSLSLSL pattern
	// like ababab... above, but then we must also arrange
	// for a large number of distinct LMS-substrings.
	// We use this pattern:
	// 1 255 1 254 1 253 1 ... 1 2 1 255 2 254 2 253 2 252 2 ...
	// This gives approximately 2¹⁵ distinct LMS-substrings.
	// We need to repeat at least one substring, though,
	// or else the recursion can be bypassed entirely.
	x := make([]byte, 100000, 100001)
	lo := byte(1)
	hi := byte(255)
	for i := range x {
	if i%2 == 0 {
	x[i] = lo
	} else {
	x[i] = hi
	hi--
	if hi <= lo {
	lo++
	if lo == 0 {
	lo = 1
	}
	hi = 255
	}
	}
	}
	x[:cap(x)][len(x)] = 0 // for sais.New
	testSA(t, x, build)
	})

	t.Run("exhaustive2", func(t *testing.T) {
	// All inputs over {0,1} up to length 21.
	// Runs in about 10 seconds on my laptop.
	x := make([]byte, 30)
	numFail := 0
	for n := 0; n <= 21; n++ {
	if n > 12 && testing.Short() {
	break
	}
	x[n] = 0 // for sais.New
	testRec(t, x[:n], 0, 2, &numFail, build)
	}
	})

	t.Run("exhaustive3", func(t *testing.T) {
	// All inputs over {0,1,2} up to length 14.
	// Runs in about 10 seconds on my laptop.
	x := make([]byte, 30)
	numFail := 0
	for n := 0; n <= 14; n++ {
	if n > 8 && testing.Short() {
	break
	}
	x[n] = 0 // for sais.New
	testRec(t, x[:n], 0, 3, &numFail, build)
	}
	})
	}

	// testRec fills x[i:] with all possible combinations of values in [1,max]
	// and then calls testSA(t, x, build) for each one.
	func testRec(t testing.T, x []byte, i, max int, numFail int, build func([]byte) []int) {
	if i < len(x) {
	for x[i] = 1; x[i] <= byte(max); x[i]++ {
	testRec(t, x, i+1, max, numFail, build)
	}
	return
	}

	if !testSA(t, x, build) {
	*numFail++
	if *numFail >= 10 {
	t.Errorf("stopping after %d failures", *numFail)
	t.FailNow()
	}
	}
	}

	// testSA tests the suffix array build function on the input x.
	// It constructs the suffix array and then checks that it is correct.
	func testSA(t *testing.T, x []byte, build func([]byte) []int) bool {
	defer func() {
	if e := recover(); e != nil {
	t.Logf("build %v", x)
	panic(e)
	}
	}()
	sa := build(x)
	if len(sa) != len(x) {
	t.Errorf("build %v: len(sa) = %d, want %d", x, len(sa), len(x))
	return false
	}
	for i := 0; i+1 < len(sa); i++ {
	if sa[i] < 0 \|\| sa[i] >= len(x) \|\| sa[i+1] < 0 \|\| sa[i+1] >= len(x) {
	t.Errorf("build %s: sa out of range: %v\n", x, sa)
	return false
	}
	if bytes.Compare(x[sa[i]:], x[sa[i+1]:]) >= 0 {
	t.Errorf("build %v -> %v\nsa[%d:] = %d,%d out of order", x, sa, i, sa[i], sa[i+1])
	return false
	}
	}

	return true
	}

	var (
	benchdata = make([]byte, 1e6)
	benchrand = make([]byte, 1e6)
	)

	// Of all possible inputs, the random bytes have the least amount of substring
	// repetition, and the repeated bytes have the most. For most algorithms,
	// the running time of every input will be between these two.
	func benchmarkNew(b *testing.B, random bool) {
	b.ReportAllocs()
	b.StopTimer()
	data := benchdata
	if random {
	data = benchrand
	if data[0] == 0 {
	for i := range data {
	data[i] = byte(rand.Intn(256))
	}
	}
	}
	b.StartTimer()
	b.SetBytes(int64(len(data)))
	for i := 0; i < b.N; i++ {
	New(data)
	}
	}

	func makeText(name string) ([]byte, error) {
	var data []byte
	switch name {
	case "opticks":
	var err error
	data, err = os.ReadFile("../../testdata/Isaac.Newton-Opticks.txt")
	if err != nil {
	return nil, err
	}
	case "go":
	err := filepath.WalkDir("../..", func(path string, info fs.DirEntry, err error) error {
	if err == nil && strings.HasSuffix(path, ".go") && !info.IsDir() {
	file, err := os.ReadFile(path)
	if err != nil {
	return err
	}
	data = append(data, file...)
	}
	return nil
	})
	if err != nil {
	return nil, err
	}
	case "zero":
	data = make([]byte, 50e6)
	case "rand":
	data = make([]byte, 50e6)
	for i := range data {
	data[i] = byte(rand.Intn(256))
	}
	}
	return data, nil
	}

	func setBits(bits int) (cleanup func()) {
	if bits == 32 {
	maxData32 = realMaxData32
	} else {
	maxData32 = -1 // force use of 64-bit code
	}
	return func() {
	maxData32 = realMaxData32
	}
	}

	func BenchmarkNew(b *testing.B) {
	for _, text := range []string{"opticks", "go", "zero", "rand"} {
	b.Run("text="+text, func(b *testing.B) {
	data, err := makeText(text)
	if err != nil {
	b.Fatal(err)
	}
	if testing.Short() && len(data) > 5e6 {
	data = data[:5e6]
	}
	for _, size := range []int{100e3, 500e3, 1e6, 5e6, 10e6, 50e6} {
	if len(data) < size {
	continue
	}
	data := data[:size]
	name := fmt.Sprintf("%dK", size/1e3)
	if size >= 1e6 {
	name = fmt.Sprintf("%dM", size/1e6)
	}
	b.Run("size="+name, func(b *testing.B) {
	for _, bits := range []int{32, 64} {
	if ^uint(0) == 0xffffffff && bits == 64 {
	continue
	}
	b.Run(fmt.Sprintf("bits=%d", bits), func(b *testing.B) {
	cleanup := setBits(bits)
	defer cleanup()

	b.SetBytes(int64(len(data)))
	b.ReportAllocs()
	for i := 0; i < b.N; i++ {
	New(data)
	}
	})
	}
	})
	}
	})
	}
	}

	func BenchmarkSaveRestore(b *testing.B) {
	r := rand.New(rand.NewSource(0x5a77a1)) // guarantee always same sequence
	data := make([]byte, 1<<20) // 1MB of data to index
	for i := range data {
	data[i] = byte(r.Intn(256))
	}
	for _, bits := range []int{32, 64} {
	if ^uint(0) == 0xffffffff && bits == 64 {
	continue
	}
	b.Run(fmt.Sprintf("bits=%d", bits), func(b *testing.B) {
	cleanup := setBits(bits)
	defer cleanup()

	b.StopTimer()
	x := New(data)
	size := testSaveRestore(nil, nil, x) // verify correctness
	buf := bytes.NewBuffer(make([]byte, size)) // avoid growing
	b.SetBytes(int64(size))
	b.StartTimer()
	b.ReportAllocs()
	for i := 0; i < b.N; i++ {
	buf.Reset()
	if err := x.Write(buf); err != nil {
	b.Fatal(err)
	}
	var y Index
	if err := y.Read(buf); err != nil {
	b.Fatal(err)
	}
	}
	})
	}
	}