storage/app: add BenchmarkReader struct
BenchmarkReader is capable of reading standard benchmark files into
Result objects and writing those Result objects back out to a Writer.
Change-Id: I022221f53b5d3ce1de7e8e7b74d265a50ac4a0eb
Reviewed-on: https://go-review.googlesource.com/34627
Reviewed-by: Russ Cox <rsc@golang.org>
diff --git a/storage/app/benchfmt.go b/storage/app/benchfmt.go
new file mode 100644
index 0000000..b115950
--- /dev/null
+++ b/storage/app/benchfmt.go
@@ -0,0 +1,227 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package app
+
+import (
+ "bufio"
+ "fmt"
+ "io"
+ "sort"
+ "strconv"
+ "strings"
+ "unicode"
+)
+
+// BenchmarkReader reads benchmark results from an io.Reader.
+type BenchmarkReader struct {
+ s *bufio.Scanner
+ labels map[string]string
+ lineNum int
+}
+
+// NewBenchmarkReader creates a BenchmarkReader that reads from r.
+func NewBenchmarkReader(r io.Reader) *BenchmarkReader {
+ return &BenchmarkReader{
+ s: bufio.NewScanner(r),
+ labels: make(map[string]string),
+ }
+}
+
+// AddLabels adds additional labels as if they had been read from the file.
+// It must be called before the first call to r.Next.
+func (r *BenchmarkReader) AddLabels(labels map[string]string) {
+ for k, v := range labels {
+ r.labels[k] = v
+ }
+}
+
+// TODO: It would probably be helpful to add a named type for
+// map[string]string with String(), Keys(), and Equal() methods.
+
+// Result represents a single line from a benchmark file.
+// All information about that line is self-contained in the Result.
+type Result struct {
+ // Labels is the set of persistent labels that apply to the result.
+ // Labels must not be modified.
+ Labels map[string]string
+ // NameLabels is the set of ephemeral labels that were parsed
+ // from the benchmark name/line.
+ // NameLabels must not be modified.
+ NameLabels map[string]string
+ // LineNum is the line number on which the result was found
+ LineNum int
+ // Content is the verbatim input line of the benchmark file, beginning with the string "Benchmark".
+ Content string
+}
+
+// A BenchmarkPrinter prints a sequence of benchmark results.
+type BenchmarkPrinter struct {
+ w io.Writer
+ labels map[string]string
+}
+
+// NewBenchmarkPrinter constructs a BenchmarkPrinter writing to w.
+func NewBenchmarkPrinter(w io.Writer) *BenchmarkPrinter {
+ return &BenchmarkPrinter{w: w}
+}
+
+// Print writes the lines necessary to recreate r.
+func (bp *BenchmarkPrinter) Print(r *Result) error {
+ var keys []string
+ // Print removed keys first.
+ for k := range bp.labels {
+ if r.Labels[k] == "" {
+ keys = append(keys, k)
+ }
+ }
+ sort.Strings(keys)
+ for _, k := range keys {
+ if _, err := fmt.Fprintf(bp.w, "%s:\n", k); err != nil {
+ return err
+ }
+ }
+ // Then print new or changed keys.
+ keys = keys[:0]
+ for k, v := range r.Labels {
+ if v != "" && bp.labels[k] != v {
+ keys = append(keys, k)
+ }
+ }
+ sort.Strings(keys)
+ for _, k := range keys {
+ if _, err := fmt.Fprintf(bp.w, "%s: %s\n", k, r.Labels[k]); err != nil {
+ return err
+ }
+ }
+ // Finally print the actual line itself.
+ if _, err := fmt.Fprintf(bp.w, "%s\n", r.Content); err != nil {
+ return err
+ }
+ bp.labels = r.Labels
+ return nil
+}
+
+// parseNameLabels extracts extra labels from a benchmark name and sets them in labels.
+func parseNameLabels(name string, labels map[string]string) {
+ dash := strings.LastIndex(name, "-")
+ if dash >= 0 {
+ // Accept -N as an alias for /GOMAXPROCS=N
+ _, err := strconv.Atoi(name[dash+1:])
+ if err == nil {
+ labels["GOMAXPROCS"] = name[dash+1:]
+ name = name[:dash]
+ }
+ }
+ parts := strings.Split(name, "/")
+ labels["name"] = parts[0]
+ for i, sub := range parts[1:] {
+ equals := strings.Index(sub, "=")
+ var key string
+ if equals >= 0 {
+ key, sub = sub[:equals], sub[equals+1:]
+ } else {
+ key = fmt.Sprintf("sub%d", i+1)
+ }
+ labels[key] = sub
+ }
+}
+
+// newResult parses a line and returns a Result object for the line.
+func newResult(labels map[string]string, lineNum int, name, content string) *Result {
+ r := &Result{
+ Labels: labels,
+ NameLabels: make(map[string]string),
+ LineNum: lineNum,
+ Content: content,
+ }
+ parseNameLabels(name, r.NameLabels)
+ return r
+}
+
+// copyLabels makes a new copy of the labels map, to protect against
+// future modifications to labels.
+func copyLabels(labels map[string]string) map[string]string {
+ new := make(map[string]string)
+ for k, v := range labels {
+ new[k] = v
+ }
+ return new
+}
+
+// TODO(quentin): How to represent and efficiently group multiple lines?
+
+// Next returns the next benchmark result from the file. If there are
+// no further results, it returns nil, io.EOF.
+func (r *BenchmarkReader) Next() (*Result, error) {
+ copied := false
+ for r.s.Scan() {
+ r.lineNum++
+ line := r.s.Text()
+ if key, value, ok := parseKeyValueLine(line); ok {
+ if !copied {
+ copied = true
+ r.labels = copyLabels(r.labels)
+ }
+ // TODO(quentin): Spec says empty value is valid, but
+ // we need a way to cancel previous labels, so we'll
+ // treat an empty value as a removal.
+ if value == "" {
+ delete(r.labels, key)
+ } else {
+ r.labels[key] = value
+ }
+ continue
+ }
+ if fullName, ok := parseBenchmarkLine(line); ok {
+ return newResult(r.labels, r.lineNum, fullName, line), nil
+ }
+ }
+ if err := r.s.Err(); err != nil {
+ return nil, err
+ }
+ return nil, io.EOF
+}
+
+// parseKeyValueLine attempts to parse line as a key: value pair. ok
+// indicates whether the line could be parsed.
+func parseKeyValueLine(line string) (key, val string, ok bool) {
+ for i, c := range line {
+ if i == 0 && !unicode.IsLower(c) {
+ return
+ }
+ if unicode.IsSpace(c) || unicode.IsUpper(c) {
+ return
+ }
+ if i > 0 && c == ':' {
+ key = line[:i]
+ val = line[i+1:]
+ break
+ }
+ }
+ if val == "" {
+ ok = true
+ return
+ }
+ for len(val) > 0 && (val[0] == ' ' || val[0] == '\t') {
+ val = val[1:]
+ ok = true
+ }
+ return
+}
+
+// parseBenchmarkLine attempts to parse line as a benchmark result. If
+// successful, fullName is the name of the benchmark with the
+// "Benchmark" prefix stripped, and ok is true.
+func parseBenchmarkLine(line string) (fullName string, ok bool) {
+ space := strings.IndexFunc(line, unicode.IsSpace)
+ if space < 0 {
+ return
+ }
+ name := line[:space]
+ if !strings.HasPrefix(name, "Benchmark") {
+ return
+ }
+ return name[len("Benchmark"):], true
+}
diff --git a/storage/app/benchfmt_test.go b/storage/app/benchfmt_test.go
new file mode 100644
index 0000000..0e9d763
--- /dev/null
+++ b/storage/app/benchfmt_test.go
@@ -0,0 +1,207 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package app
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "reflect"
+ "strings"
+ "testing"
+)
+
+func readAllResults(t *testing.T, r *BenchmarkReader) []*Result {
+ var out []*Result
+ for {
+ result, err := r.Next()
+ switch err {
+ case io.EOF:
+ return out
+ case nil:
+ out = append(out, result)
+ default:
+ t.Fatal(err)
+ }
+ }
+}
+
+func TestBenchmarkReader(t *testing.T) {
+ type kv map[string]string
+ tests := []struct {
+ name, input string
+ want []*Result
+ }{
+ {
+ "basic",
+ `key: value
+BenchmarkOne 1 ns/sec
+`,
+ []*Result{{
+ kv{"key": "value"},
+ kv{"name": "One"},
+ 2,
+ "BenchmarkOne 1 ns/sec",
+ }},
+ },
+ {
+ "two results with indexed and named subnames",
+ `key: value
+BenchmarkOne/foo/bar=1-2 1 ns/sec
+BenchmarkTwo 2 ns/sec
+`,
+ []*Result{
+ {
+ kv{"key": "value"},
+ kv{"name": "One", "sub1": "foo", "bar": "1", "GOMAXPROCS": "2"},
+ 2,
+ "BenchmarkOne/foo/bar=1-2 1 ns/sec",
+ },
+ {
+ kv{"key": "value"},
+ kv{"name": "Two"},
+ 3,
+ "BenchmarkTwo 2 ns/sec",
+ },
+ },
+ },
+ {
+ "remove existing label",
+ `key: value
+key:
+BenchmarkOne 1 ns/sec
+`,
+ []*Result{
+ {
+ kv{},
+ kv{"name": "One"},
+ 3,
+ "BenchmarkOne 1 ns/sec",
+ },
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ r := NewBenchmarkReader(strings.NewReader(test.input))
+ have := readAllResults(t, r)
+ want := test.want
+ diff := ""
+ mismatch := false
+ for i := 0; i < len(have) || i < len(want); i++ {
+ if i < len(have) && i < len(want) && reflect.DeepEqual(have[i], want[i]) {
+ diff += fmt.Sprintf(" %+v\n", have[i])
+ continue
+ }
+ mismatch = true
+ if i < len(have) {
+ diff += fmt.Sprintf("-%+v\n", have[i])
+ }
+ if i < len(want) {
+ diff += fmt.Sprintf("+%+v\n", want[i])
+ }
+ }
+ if mismatch {
+ t.Errorf("wrong results: (- have/+ want)\n%s", diff)
+ }
+ })
+ }
+}
+
+func TestBenchmarkPrinter(t *testing.T) {
+ tests := []struct {
+ name, input, want string
+ }{
+ {
+ "basic",
+ `key: value
+BenchmarkOne 1 ns/sec
+`,
+ `key: value
+BenchmarkOne 1 ns/sec
+`,
+ },
+ {
+ "missing newline",
+ `key: value
+BenchmarkOne 1 ns/sec`,
+ `key: value
+BenchmarkOne 1 ns/sec
+`,
+ },
+ {
+ "duplicate and removed fields",
+ `one: 1
+two: 2
+BenchmarkOne 1 ns/sec
+one: 1
+two: 3
+BenchmarkOne 1 ns/sec
+two:
+BenchmarkOne 1 ns/sec
+`,
+ `one: 1
+two: 2
+BenchmarkOne 1 ns/sec
+two: 3
+BenchmarkOne 1 ns/sec
+two:
+BenchmarkOne 1 ns/sec
+`,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ r := NewBenchmarkReader(strings.NewReader(test.input))
+ results := readAllResults(t, r)
+ var have bytes.Buffer
+ bp := NewBenchmarkPrinter(&have)
+ for _, result := range results {
+ if err := bp.Print(result); err != nil {
+ t.Errorf("Print returned %v", err)
+ }
+ }
+ if diff := diff(have.String(), test.want); diff != "" {
+ t.Errorf("wrong output: (- got/+ want)\n%s", diff)
+ }
+ })
+ }
+}
+
+// diff returns the output of unified diff on s1 and s2. If the result
+// is non-empty, the strings differ or the diff command failed.
+func diff(s1, s2 string) string {
+ f1, err := ioutil.TempFile("", "benchfmt_test")
+ if err != nil {
+ return err.Error()
+ }
+ defer os.Remove(f1.Name())
+ defer f1.Close()
+
+ f2, err := ioutil.TempFile("", "benchfmt_test")
+ if err != nil {
+ return err.Error()
+ }
+ defer os.Remove(f2.Name())
+ defer f2.Close()
+
+ f1.Write([]byte(s1))
+ f2.Write([]byte(s2))
+
+ data, err := exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput()
+ if len(data) > 0 {
+ // diff exits with a non-zero status when the files don't match.
+ // Ignore that failure as long as we get output.
+ err = nil
+ }
+ if err != nil {
+ data = append(data, []byte(err.Error())...)
+ }
+ return string(data)
+
+}
diff --git a/storage/app/upload.go b/storage/app/upload.go
index 1f968a2..7943f9e 100644
--- a/storage/app/upload.go
+++ b/storage/app/upload.go
@@ -10,6 +10,7 @@
"io"
"mime/multipart"
"net/http"
+ "sort"
"golang.org/x/net/context"
)
@@ -102,7 +103,18 @@
return nil, err
}
- // TODO(quentin): Write metadata at top of file
+ var keys []string
+ for k := range meta {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ for _, k := range keys {
+ if _, err := fmt.Fprintf(fw, "%s: %s\n", k, meta[k]); err != nil {
+ fw.CloseWithError(err)
+ return nil, err
+ }
+ }
+
if _, err := io.Copy(fw, p); err != nil {
fw.CloseWithError(err)
return nil, err
