benchproc/projection_test.go - perf - Git at Google

 // Copyright 2022 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 package benchproc

 import (
 	"fmt"
 	"strings"
 	"testing"

 	"golang.org/x/perf/benchfmt"
 	"golang.org/x/perf/benchproc/internal/parse"
 )

 // mustParse parses a single projection.
 func mustParse(t *testing.T, proj string) (*Projection, *Filter) {
 	f, err := NewFilter("*")
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
 	s, err := (&ProjectionParser{}).Parse(proj, f)
 	if err != nil {
 		t.Fatalf("unexpected error parsing %q: %v", proj, err)
 	}
 	return s, f
 }

 // r constructs a benchfmt.Result with the given full name and file
 // config, which is specified as alternating key/value pairs. The
 // result has 1 iteration and no values.
 func r(t *testing.T, fullName string, fileConfig ...string) *benchfmt.Result {
 	res := &benchfmt.Result{
 		Name:  benchfmt.Name(fullName),
 		Iters: 1,
 	}

 	if len(fileConfig)%2 != 0 {
 		t.Fatal("fileConfig must be alternating key/value pairs")
 	}
 	for i := 0; i < len(fileConfig); i += 2 {
 		cfg := benchfmt.Config{Key: fileConfig[i], Value: []byte(fileConfig[i+1]), File: true}
 		res.Config = append(res.Config, cfg)
 	}

 	return res
 }

 // p constructs a benchfmt.Result like r, then projects it using p.
 func p(t *testing.T, p *Projection, fullName string, fileConfig ...string) Key {
 	res := r(t, fullName, fileConfig...)
 	return p.Project(res)
 }

 func TestProjectionBasic(t *testing.T) {
 	check := func(key Key, want string) {
 		t.Helper()
 		got := key.String()
 		if got != want {
 			t.Errorf("got %s, want %s", got, want)
 		}
 	}

 	var s *Projection

 	// Sub-name config.
 	s, _ = mustParse(t, ".fullname")
 	check(p(t, s, "Name/a=1"), ".fullname:Name/a=1")
 	s, _ = mustParse(t, "/a")
 	check(p(t, s, "Name/a=1"), "/a:1")
 	s, _ = mustParse(t, ".name")
 	check(p(t, s, "Name/a=1"), ".name:Name")

 	// Fixed file config.
 	s, _ = mustParse(t, "a")
 	check(p(t, s, "", "a", "1", "b", "2"), "a:1")
 	check(p(t, s, "", "b", "2"), "") // Missing values are omitted
 	check(p(t, s, "", "a", "", "b", "2"), "")

 	// Variable file config.
 	s, _ = mustParse(t, ".config")
 	check(p(t, s, "", "a", "1", "b", "2"), "a:1 b:2")
 	check(p(t, s, "", "c", "3"), "c:3")
 	check(p(t, s, "", "c", "3", "a", "2"), "a:2 c:3")
 }

 func TestProjectionIntern(t *testing.T) {
 	s, _ := mustParse(t, "a,b")

 	c12 := p(t, s, "", "a", "1", "b", "2")

 	if c12 != p(t, s, "", "a", "1", "b", "2") {
 		t.Errorf("Keys should be equal")
 	}

 	if c12 == p(t, s, "", "a", "1", "b", "3") {
 		t.Errorf("Keys should not be equal")
 	}

 	if c12 != p(t, s, "", "a", "1", "b", "2", "c", "3") {
 		t.Errorf("Keys should be equal")
 	}
 }

 func fieldNames(fields []*Field) string {
 	names := new(strings.Builder)
 	for i, f := range fields {
 		if i > 0 {
 			names.WriteByte(' ')
 		}
 		names.WriteString(f.Name)
 	}
 	return names.String()
 }

 func TestProjectionParsing(t *testing.T) {
 	// Basic parsing is tested by the syntax package. Here we test
 	// additional processing done by this package.

 	check := func(proj string, want, wantFlat string) {
 		t.Helper()
 		s, _ := mustParse(t, proj)
 		got := fieldNames(s.Fields())
 		if got != want {
 			t.Errorf("%s: got fields %v, want %v", proj, got, want)
 		}
 		gotFlat := fieldNames(s.FlattenedFields())
 		if gotFlat != wantFlat {
 			t.Errorf("%s: got flat fields %v, want %v", proj, gotFlat, wantFlat)
 		}
 	}
 	checkErr := func(proj, error string, pos int) {
 		t.Helper()
 		f, _ := NewFilter("*")
 		_, err := (&ProjectionParser{}).Parse(proj, f)
 		if se, _ := err.(*parse.SyntaxError); se == nil || se.Msg != error || se.Off != pos {
 			t.Errorf("%s: want error %s at %d; got %s", proj, error, pos, err)
 		}
 	}

 	check("a,b,c", "a b c", "a b c")
 	check("a,.config,c", "a .config c", "a c") // .config hasn't been populated with anything yet
 	check("a,.fullname,c", "a .fullname c", "a .fullname c")
 	check("a,.name,c", "a .name c", "a .name c")
 	check("a,/b,c", "a /b c", "a /b c")

 	checkErr("a@foo", "unknown order \"foo\"", 2)

 	checkErr(".config@(1 2)", "fixed order not allowed for .config", 8)
 }

 func TestProjectionFiltering(t *testing.T) {
 	_, f := mustParse(t, "a@(a b c)")
 	check := func(val string, want bool) {
 		t.Helper()
 		res := r(t, "", "a", val)
 		got, _ := f.Apply(res)
 		if want != got {
 			t.Errorf("%s: want %v, got %v", val, want, got)
 		}
 	}
 	check("a", true)
 	check("b", true)
 	check("aa", false)
 	check("z", false)
 }

 func TestProjectionExclusion(t *testing.T) {
 	// The underlying name normalization has already been tested
 	// thoroughly in benchfmt/extract_test.go, so here we just
 	// have to test that it's being plumbed right.

 	check := func(key Key, want string) {
 		t.Helper()
 		got := key.String()
 		if got != want {
 			t.Errorf("got %s, want %s", got, want)
 		}
 	}

 	// Create the main projection.
 	var pp ProjectionParser
 	f, _ := NewFilter("*")
 	s, err := pp.Parse(".fullname,.config", f)
 	if err != nil {
 		t.Fatalf("unexpected error %v", err)
 	}
 	// Parse specific keys that should be excluded from fullname
 	// and config.
 	_, err = pp.Parse(".name,/a,exc", f)
 	if err != nil {
 		t.Fatalf("unexpected error %v", err)
 	}

 	check(p(t, s, "Name"), ".fullname:*")
 	check(p(t, s, "Name/a=1"), ".fullname:*")
 	check(p(t, s, "Name/a=1/b=2"), ".fullname:*/b=2")

 	check(p(t, s, "Name", "exc", "1"), ".fullname:*")
 	check(p(t, s, "Name", "exc", "1", "abc", "2"), ".fullname:* abc:2")
 	check(p(t, s, "Name", "abc", "2"), ".fullname:* abc:2")
 }

 func TestProjectionResidue(t *testing.T) {
 	check := func(mainProj string, want string) {
 		t.Helper()

 		// Get the residue of mainProj.
 		var pp ProjectionParser
 		f, _ := NewFilter("*")
 		_, err := pp.Parse(mainProj, f)
 		if err != nil {
 			t.Fatalf("unexpected error %v", err)
 		}
 		s := pp.Residue()

 		// Project a test result.
 		key := p(t, s, "Name/a=1/b=2", "x", "3", "y", "4")
 		got := key.String()
 		if got != want {
 			t.Errorf("got %s, want %s", got, want)
 		}
 	}

 	// Full residue.
 	check("", "x:3 y:4 .fullname:Name/a=1/b=2")
 	// Empty residue.
 	check(".config,.fullname", "")
 	// Partial residues.
 	check("x,/a", "y:4 .fullname:Name/b=2")
 	check(".config", ".fullname:Name/a=1/b=2")
 	check(".fullname", "x:3 y:4")
 	check(".name", "x:3 y:4 .fullname:*/a=1/b=2")
 }

 // ExampleProjectionParser_Residue demonstrates residue projections.
 //
 // This example groups a set of results by .fullname and goos, but the
 // two results for goos:linux have two different goarch values,
 // indicating the user probably unintentionally grouped uncomparable
 // results together. The example uses ProjectionParser.Residue and
 // NonSingularFields to warn the user about this.
 func ExampleProjectionParser_Residue() {
 	var pp ProjectionParser
 	p, _ := pp.Parse(".fullname,goos", nil)
 	residue := pp.Residue()

 	// Aggregate each result by p and track the residue of each group.
 	type group struct {
 		values   []float64
 		residues []Key
 	}
 	groups := make(map[Key]*group)
 	var keys []Key

 	for _, result := range results(`
 goos: linux
 goarch: amd64
 BenchmarkAlloc 1 128 ns/op

 goos: linux
 goarch: arm64
 BenchmarkAlloc 1 137 ns/op

 goos: darwin
 goarch: amd64
 BenchmarkAlloc 1 130 ns/op`) {
 		// Map result to a group.
 		key := p.Project(result)
 		g, ok := groups[key]
 		if !ok {
 			g = new(group)
 			groups[key] = g
 			keys = append(keys, key)
 		}

 		// Add value to the group.
 		speed, _ := result.Value("sec/op")
 		g.values = append(g.values, speed)

 		// Add residue to the group.
 		g.residues = append(g.residues, residue.Project(result))
 	}

 	// Report aggregated results.
 	SortKeys(keys)
 	for _, k := range keys {
 		g := groups[k]
 		// Report the result.
 		fmt.Println(k, mean(g.values), "sec/op")
 		// Check if the grouped results vary in some unexpected way.
 		nonsingular := NonSingularFields(g.residues)
 		if len(nonsingular) > 0 {
 			// Report a potential issue.
 			fmt.Printf("warning: results vary in %s and may be uncomparable\n", nonsingular)
 		}
 	}

 	// Output:
 	// .fullname:Alloc goos:linux 1.325e-07 sec/op
 	// warning: results vary in [goarch] and may be uncomparable
 	// .fullname:Alloc goos:darwin 1.3e-07 sec/op
 }

 func results(data string) []*benchfmt.Result {
 	var out []*benchfmt.Result
 	r := benchfmt.NewReader(strings.NewReader(data), "<string>")
 	for r.Scan() {
 		switch rec := r.Result(); rec := rec.(type) {
 		case *benchfmt.Result:
 			out = append(out, rec.Clone())
 		case *benchfmt.SyntaxError:
 			panic("unexpected error in test data: " + rec.Error())
 		}
 	}
 	return out
 }

 func mean(xs []float64) float64 {
 	var sum float64
 	for _, x := range xs {
 		sum += x
 	}
 	return sum / float64(len(xs))
 }

 func TestProjectionValues(t *testing.T) {
 	s, unit, err := (&ProjectionParser{}).ParseWithUnit("x", nil)
 	if err != nil {
 		t.Fatalf("unexpected error parsing %q: %v", "x", err)
 	}

 	check := func(key Key, want, wantUnit string) {
 		t.Helper()
 		got := key.String()
 		if got != want {
 			t.Errorf("got %s, want %s", got, want)
 		}
 		gotUnit := key.Get(unit)
 		if gotUnit != wantUnit {
 			t.Errorf("got unit %s, want %s", gotUnit, wantUnit)
 		}
 	}

 	res := r(t, "Name", "x", "1")
 	res.Values = []benchfmt.Value{{Value: 100, Unit: "ns/op"}, {Value: 1.21, Unit: "gigawatts"}}
 	keys := s.ProjectValues(res)
 	if len(keys) != len(res.Values) {
 		t.Fatalf("got %d Keys, want %d", len(keys), len(res.Values))
 	}

 	check(keys[0], "x:1 .unit:ns/op", "ns/op")
 	check(keys[1], "x:1 .unit:gigawatts", "gigawatts")
 }
	// Copyright 2022 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	package benchproc

	import (
	"fmt"
	"strings"
	"testing"

	"golang.org/x/perf/benchfmt"
	"golang.org/x/perf/benchproc/internal/parse"
	)

	// mustParse parses a single projection.
	func mustParse(t testing.T, proj string) (Projection, *Filter) {
	f, err := NewFilter("*")
	if err != nil {
	t.Fatalf("unexpected error: %v", err)
	}
	s, err := (&ProjectionParser{}).Parse(proj, f)
	if err != nil {
	t.Fatalf("unexpected error parsing %q: %v", proj, err)
	}
	return s, f
	}

	// r constructs a benchfmt.Result with the given full name and file
	// config, which is specified as alternating key/value pairs. The
	// result has 1 iteration and no values.
	func r(t testing.T, fullName string, fileConfig ...string) benchfmt.Result {
	res := &benchfmt.Result{
	Name: benchfmt.Name(fullName),
	Iters: 1,
	}

	if len(fileConfig)%2 != 0 {
	t.Fatal("fileConfig must be alternating key/value pairs")
	}
	for i := 0; i < len(fileConfig); i += 2 {
	cfg := benchfmt.Config{Key: fileConfig[i], Value: []byte(fileConfig[i+1]), File: true}
	res.Config = append(res.Config, cfg)
	}

	return res
	}

	// p constructs a benchfmt.Result like r, then projects it using p.
	func p(t testing.T, p Projection, fullName string, fileConfig ...string) Key {
	res := r(t, fullName, fileConfig...)
	return p.Project(res)
	}

	func TestProjectionBasic(t *testing.T) {
	check := func(key Key, want string) {
	t.Helper()
	got := key.String()
	if got != want {
	t.Errorf("got %s, want %s", got, want)
	}
	}

	var s *Projection

	// Sub-name config.
	s, _ = mustParse(t, ".fullname")
	check(p(t, s, "Name/a=1"), ".fullname:Name/a=1")
	s, _ = mustParse(t, "/a")
	check(p(t, s, "Name/a=1"), "/a:1")
	s, _ = mustParse(t, ".name")
	check(p(t, s, "Name/a=1"), ".name:Name")

	// Fixed file config.
	s, _ = mustParse(t, "a")
	check(p(t, s, "", "a", "1", "b", "2"), "a:1")
	check(p(t, s, "", "b", "2"), "") // Missing values are omitted
	check(p(t, s, "", "a", "", "b", "2"), "")

	// Variable file config.
	s, _ = mustParse(t, ".config")
	check(p(t, s, "", "a", "1", "b", "2"), "a:1 b:2")
	check(p(t, s, "", "c", "3"), "c:3")
	check(p(t, s, "", "c", "3", "a", "2"), "a:2 c:3")
	}

	func TestProjectionIntern(t *testing.T) {
	s, _ := mustParse(t, "a,b")

	c12 := p(t, s, "", "a", "1", "b", "2")

	if c12 != p(t, s, "", "a", "1", "b", "2") {
	t.Errorf("Keys should be equal")
	}

	if c12 == p(t, s, "", "a", "1", "b", "3") {
	t.Errorf("Keys should not be equal")
	}

	if c12 != p(t, s, "", "a", "1", "b", "2", "c", "3") {
	t.Errorf("Keys should be equal")
	}
	}

	func fieldNames(fields []*Field) string {
	names := new(strings.Builder)
	for i, f := range fields {
	if i > 0 {
	names.WriteByte(' ')
	}
	names.WriteString(f.Name)
	}
	return names.String()
	}

	func TestProjectionParsing(t *testing.T) {
	// Basic parsing is tested by the syntax package. Here we test
	// additional processing done by this package.

	check := func(proj string, want, wantFlat string) {
	t.Helper()
	s, _ := mustParse(t, proj)
	got := fieldNames(s.Fields())
	if got != want {
	t.Errorf("%s: got fields %v, want %v", proj, got, want)
	}
	gotFlat := fieldNames(s.FlattenedFields())
	if gotFlat != wantFlat {
	t.Errorf("%s: got flat fields %v, want %v", proj, gotFlat, wantFlat)
	}
	}
	checkErr := func(proj, error string, pos int) {
	t.Helper()
	f, _ := NewFilter("*")
	_, err := (&ProjectionParser{}).Parse(proj, f)
	if se, _ := err.(*parse.SyntaxError); se == nil \|\| se.Msg != error \|\| se.Off != pos {
	t.Errorf("%s: want error %s at %d; got %s", proj, error, pos, err)
	}
	}

	check("a,b,c", "a b c", "a b c")
	check("a,.config,c", "a .config c", "a c") // .config hasn't been populated with anything yet
	check("a,.fullname,c", "a .fullname c", "a .fullname c")
	check("a,.name,c", "a .name c", "a .name c")
	check("a,/b,c", "a /b c", "a /b c")

	checkErr("a@foo", "unknown order \"foo\"", 2)

	checkErr(".config@(1 2)", "fixed order not allowed for .config", 8)
	}

	func TestProjectionFiltering(t *testing.T) {
	_, f := mustParse(t, "a@(a b c)")
	check := func(val string, want bool) {
	t.Helper()
	res := r(t, "", "a", val)
	got, _ := f.Apply(res)
	if want != got {
	t.Errorf("%s: want %v, got %v", val, want, got)
	}
	}
	check("a", true)
	check("b", true)
	check("aa", false)
	check("z", false)
	}

	func TestProjectionExclusion(t *testing.T) {
	// The underlying name normalization has already been tested
	// thoroughly in benchfmt/extract_test.go, so here we just
	// have to test that it's being plumbed right.

	check := func(key Key, want string) {
	t.Helper()
	got := key.String()
	if got != want {
	t.Errorf("got %s, want %s", got, want)
	}
	}

	// Create the main projection.
	var pp ProjectionParser
	f, _ := NewFilter("*")
	s, err := pp.Parse(".fullname,.config", f)
	if err != nil {
	t.Fatalf("unexpected error %v", err)
	}
	// Parse specific keys that should be excluded from fullname
	// and config.
	_, err = pp.Parse(".name,/a,exc", f)
	if err != nil {
	t.Fatalf("unexpected error %v", err)
	}

	check(p(t, s, "Name"), ".fullname:*")
	check(p(t, s, "Name/a=1"), ".fullname:*")
	check(p(t, s, "Name/a=1/b=2"), ".fullname:*/b=2")

	check(p(t, s, "Name", "exc", "1"), ".fullname:*")
	check(p(t, s, "Name", "exc", "1", "abc", "2"), ".fullname:* abc:2")
	check(p(t, s, "Name", "abc", "2"), ".fullname:* abc:2")
	}

	func TestProjectionResidue(t *testing.T) {
	check := func(mainProj string, want string) {
	t.Helper()

	// Get the residue of mainProj.
	var pp ProjectionParser
	f, _ := NewFilter("*")
	_, err := pp.Parse(mainProj, f)
	if err != nil {
	t.Fatalf("unexpected error %v", err)
	}
	s := pp.Residue()

	// Project a test result.
	key := p(t, s, "Name/a=1/b=2", "x", "3", "y", "4")
	got := key.String()
	if got != want {
	t.Errorf("got %s, want %s", got, want)
	}
	}

	// Full residue.
	check("", "x:3 y:4 .fullname:Name/a=1/b=2")
	// Empty residue.
	check(".config,.fullname", "")
	// Partial residues.
	check("x,/a", "y:4 .fullname:Name/b=2")
	check(".config", ".fullname:Name/a=1/b=2")
	check(".fullname", "x:3 y:4")
	check(".name", "x:3 y:4 .fullname:*/a=1/b=2")
	}

	// ExampleProjectionParser_Residue demonstrates residue projections.
	//
	// This example groups a set of results by .fullname and goos, but the
	// two results for goos:linux have two different goarch values,
	// indicating the user probably unintentionally grouped uncomparable
	// results together. The example uses ProjectionParser.Residue and
	// NonSingularFields to warn the user about this.
	func ExampleProjectionParser_Residue() {
	var pp ProjectionParser
	p, _ := pp.Parse(".fullname,goos", nil)
	residue := pp.Residue()

	// Aggregate each result by p and track the residue of each group.
	type group struct {
	values []float64
	residues []Key
	}
	groups := make(map[Key]*group)
	var keys []Key

	for _, result := range results(`
	goos: linux
	goarch: amd64
	BenchmarkAlloc 1 128 ns/op

	goos: linux
	goarch: arm64
	BenchmarkAlloc 1 137 ns/op

	goos: darwin
	goarch: amd64
	BenchmarkAlloc 1 130 ns/op`) {
	// Map result to a group.
	key := p.Project(result)
	g, ok := groups[key]
	if !ok {
	g = new(group)
	groups[key] = g
	keys = append(keys, key)
	}

	// Add value to the group.
	speed, _ := result.Value("sec/op")
	g.values = append(g.values, speed)

	// Add residue to the group.
	g.residues = append(g.residues, residue.Project(result))
	}

	// Report aggregated results.
	SortKeys(keys)
	for _, k := range keys {
	g := groups[k]
	// Report the result.
	fmt.Println(k, mean(g.values), "sec/op")
	// Check if the grouped results vary in some unexpected way.
	nonsingular := NonSingularFields(g.residues)
	if len(nonsingular) > 0 {
	// Report a potential issue.
	fmt.Printf("warning: results vary in %s and may be uncomparable\n", nonsingular)
	}
	}

	// Output:
	// .fullname:Alloc goos:linux 1.325e-07 sec/op
	// warning: results vary in [goarch] and may be uncomparable
	// .fullname:Alloc goos:darwin 1.3e-07 sec/op
	}

	func results(data string) []*benchfmt.Result {
	var out []*benchfmt.Result
	r := benchfmt.NewReader(strings.NewReader(data), "<string>")
	for r.Scan() {
	switch rec := r.Result(); rec := rec.(type) {
	case *benchfmt.Result:
	out = append(out, rec.Clone())
	case *benchfmt.SyntaxError:
	panic("unexpected error in test data: " + rec.Error())
	}
	}
	return out
	}

	func mean(xs []float64) float64 {
	var sum float64
	for _, x := range xs {
	sum += x
	}
	return sum / float64(len(xs))
	}

	func TestProjectionValues(t *testing.T) {
	s, unit, err := (&ProjectionParser{}).ParseWithUnit("x", nil)
	if err != nil {
	t.Fatalf("unexpected error parsing %q: %v", "x", err)
	}

	check := func(key Key, want, wantUnit string) {
	t.Helper()
	got := key.String()
	if got != want {
	t.Errorf("got %s, want %s", got, want)
	}
	gotUnit := key.Get(unit)
	if gotUnit != wantUnit {
	t.Errorf("got unit %s, want %s", gotUnit, wantUnit)
	}
	}

	res := r(t, "Name", "x", "1")
	res.Values = []benchfmt.Value{{Value: 100, Unit: "ns/op"}, {Value: 1.21, Unit: "gigawatts"}}
	keys := s.ProjectValues(res)
	if len(keys) != len(res.Values) {
	t.Fatalf("got %d Keys, want %d", len(keys), len(res.Values))
	}

	check(keys[0], "x:1 .unit:ns/op", "ns/op")
	check(keys[1], "x:1 .unit:gigawatts", "gigawatts")
	}