blob: b1bad8707969985d3176655e59168be669f6667e [file] [log] [blame]
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package benchfmt
import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
)
func parseAll(t *testing.T, data string, setup ...func(r *Reader, sr io.Reader)) ([]Record, *Reader) {
sr := strings.NewReader(data)
r := NewReader(sr, "test")
for _, f := range setup {
f(r, sr)
}
var out []Record
for r.Scan() {
switch rec := r.Result(); rec := rec.(type) {
case *Result:
res := rec.Clone()
// Wipe position information for comparisons.
res.fileName = ""
res.line = 0
out = append(out, res)
case *SyntaxError, *UnitMetadata:
out = append(out, rec)
default:
t.Fatalf("unexpected result type %T", rec)
}
}
if err := r.Err(); err != nil {
t.Fatal("parsing failed: ", err)
}
return out, r
}
func printRecord(w io.Writer, r Record) {
switch r := r.(type) {
case *Result:
for _, fc := range r.Config {
fmt.Fprintf(w, "{%s: %s} ", fc.Key, fc.Value)
}
fmt.Fprintf(w, "%s %d", r.Name.Full(), r.Iters)
for _, val := range r.Values {
fmt.Fprintf(w, " %v %s", val.Value, val.Unit)
}
fmt.Fprintf(w, "\n")
case *UnitMetadata:
fmt.Fprintf(w, "Unit: %+v\n", r)
case *SyntaxError:
fmt.Fprintf(w, "SyntaxError: %s\n", r)
default:
panic(fmt.Sprintf("unknown record type %T", r))
}
}
type resultBuilder struct {
res *Result
}
func r(fullName string, iters int) *resultBuilder {
return &resultBuilder{
&Result{
Config: []Config{},
Name: Name(fullName),
Iters: iters,
},
}
}
func (b *resultBuilder) config(keyVals ...string) *resultBuilder {
for i := 0; i < len(keyVals); i += 2 {
key, val := keyVals[i], keyVals[i+1]
file := true
if val[0] == '*' {
file = false
val = val[1:]
}
b.res.Config = append(b.res.Config, Config{key, []byte(val), file})
}
return b
}
func (b *resultBuilder) v(value float64, unit string) *resultBuilder {
var v Value
if unit == "ns/op" {
v = Value{Value: value * 1e-9, Unit: "sec/op", OrigValue: value, OrigUnit: unit}
} else {
v = Value{Value: value, Unit: unit}
}
b.res.Values = append(b.res.Values, v)
return b
}
func compareRecords(t *testing.T, got, want []Record) {
t.Helper()
var diff bytes.Buffer
for i := 0; i < len(got) || i < len(want); i++ {
if i >= len(got) {
fmt.Fprintf(&diff, "[%d] got: none, want:\n", i)
printRecord(&diff, want[i])
} else if i >= len(want) {
fmt.Fprintf(&diff, "[%d] want: none, got:\n", i)
printRecord(&diff, got[i])
} else if !reflect.DeepEqual(got[i], want[i]) {
fmt.Fprintf(&diff, "[%d] got:\n", i)
printRecord(&diff, got[i])
fmt.Fprintf(&diff, "[%d] want:\n", i)
printRecord(&diff, want[i])
}
}
if diff.Len() != 0 {
t.Error(diff.String())
}
}
func TestReader(t *testing.T) {
type testCase struct {
name, input string
want []Record
}
for _, test := range []testCase{
{
"basic",
`key: value
BenchmarkOne 100 1 ns/op 2 B/op
BenchmarkTwo 300 4.5 ns/op
`,
[]Record{
r("One", 100).
config("key", "value").
v(1, "ns/op").v(2, "B/op").res,
r("Two", 300).
config("key", "value").
v(4.5, "ns/op").res,
},
},
{
"weird",
`
BenchmarkSpaces 1 1 ns/op
BenchmarkHugeVal 1 9999999999999999999999999999999 ns/op
BenchmarkEmSpace  1  1  ns/op
`,
[]Record{
r("Spaces", 1).
v(1, "ns/op").res,
r("HugeVal", 1).
v(9999999999999999999999999999999, "ns/op").res,
r("EmSpace", 1).
v(1, "ns/op").res,
},
},
{
"basic file keys",
`key1: value
: not a key
ab:not a key
a b: also not a key
key2: value
BenchmarkOne 100 1 ns/op
`,
[]Record{
r("One", 100).
config("key1", "value", "key2", "value").
v(1, "ns/op").res,
},
},
{
"bad lines",
`not a benchmark
BenchmarkMissingIter
BenchmarkBadIter abc
BenchmarkHugeIter 9999999999999999999999999999999
BenchmarkMissingVal 100
BenchmarkBadVal 100 abc
BenchmarkMissingUnit 100 1
BenchmarkMissingUnit2 100 1 ns/op 2
also not a benchmark
Unit
Unit ns/op blah
Unit ns/op a=1
Unit ns/op a=2
`,
[]Record{
&SyntaxError{"test", 2, "missing iteration count"},
&SyntaxError{"test", 3, "parsing iteration count: invalid syntax"},
&SyntaxError{"test", 4, "parsing iteration count: value out of range"},
&SyntaxError{"test", 5, "missing measurements"},
&SyntaxError{"test", 6, "parsing measurement: invalid syntax"},
&SyntaxError{"test", 7, "missing units"},
&SyntaxError{"test", 8, "missing units"},
&SyntaxError{"test", 10, "missing unit"},
&SyntaxError{"test", 11, "expected key=value"},
&UnitMetadata{UnitMetadataKey{"sec/op", "a"}, "ns/op", "1", "test", 12},
&SyntaxError{"test", 13, "metadata a of unit ns/op already set to 1"},
},
},
{
"remove existing label",
`key: value
key:
BenchmarkOne 100 1 ns/op
`,
[]Record{
r("One", 100).
v(1, "ns/op").res,
},
},
{
"overwrite exiting label",
`key1: first
key2: second
key1: third
BenchmarkOne 100 1 ns/op
`,
[]Record{
r("One", 100).
config("key1", "third", "key2", "second").
v(1, "ns/op").res,
},
},
{
"unit metadata",
`Unit ns/op a=1 b=2
Unit ns/op c=3 error d=4
# Repeated unit should report nothing
Unit ns/op d=4
# Starts like a unit line but actually isn't
Unitx
BenchmarkOne 100 1 ns/op
`,
[]Record{
&UnitMetadata{UnitMetadataKey{"sec/op", "a"}, "ns/op", "1", "test", 1},
&UnitMetadata{UnitMetadataKey{"sec/op", "b"}, "ns/op", "2", "test", 1},
&UnitMetadata{UnitMetadataKey{"sec/op", "c"}, "ns/op", "3", "test", 2},
&SyntaxError{"test", 2, "expected key=value"},
&UnitMetadata{UnitMetadataKey{"sec/op", "d"}, "ns/op", "4", "test", 2},
r("One", 100).
v(1, "ns/op").res,
},
},
} {
t.Run(test.name, func(t *testing.T) {
got, _ := parseAll(t, test.input)
compareRecords(t, got, test.want)
})
}
}
func TestReaderInternalConfig(t *testing.T) {
got, _ := parseAll(t, `
# Test initial internal config
Benchmark1 100 1 ns/op
# Overwrite internal config with file config
key1: file1
key3: file3
Benchmark2 100 1 ns/op
# Delete internal config, check that file config is right
key2:
Benchmark3 100 1 ns/op
`, func(r *Reader, sr io.Reader) {
r.Reset(sr, "test", "key1", "internal1", "key2", "internal2")
})
want := []Record{
r("1", 100).v(1, "ns/op").config("key1", "*internal1", "key2", "*internal2").res,
r("2", 100).v(1, "ns/op").config("key1", "file1", "key2", "*internal2", "key3", "file3").res,
r("3", 100).v(1, "ns/op").config("key1", "file1", "key3", "file3").res,
}
compareRecords(t, got, want)
}
func BenchmarkReader(b *testing.B) {
path := "testdata/bent"
fileInfos, err := os.ReadDir(path)
if err != nil {
b.Fatal("reading test data directory: ", err)
}
var files []*os.File
for _, info := range fileInfos {
f, err := os.Open(filepath.Join(path, info.Name()))
if err != nil {
b.Fatal(err)
}
defer f.Close()
files = append(files, f)
}
b.ResetTimer()
start := time.Now()
var n int
for i := 0; i < b.N; i++ {
r := new(Reader)
for _, f := range files {
if _, err := f.Seek(0, 0); err != nil {
b.Fatal("seeking to 0: ", err)
}
r.Reset(f, f.Name())
for r.Scan() {
n++
if err, ok := r.Result().(error); ok {
b.Fatal("malformed record: ", err)
}
}
if err := r.Err(); err != nil {
b.Fatal(err)
}
}
}
dur := time.Since(start)
b.Logf("read %d records", n)
b.StopTimer()
b.ReportMetric(float64(n/b.N), "records/op")
b.ReportMetric(float64(n)*float64(time.Second)/float64(dur), "records/sec")
}