storage: serve summary of uploads on /uploads
This will eventually be used to serve a "Recent Uploads" section on
the analysis server, and potentially a "My Uploads" section.
Change-Id: I532fc79310d6f98ff12c80a628606845b40ceb60
Reviewed-on: https://go-review.googlesource.com/36012
Reviewed-by: Russ Cox <rsc@golang.org>
diff --git a/storage/app/app.go b/storage/app/app.go
index af6698a..558ff47 100644
--- a/storage/app/app.go
+++ b/storage/app/app.go
@@ -41,9 +41,10 @@
mux.HandleFunc("/", a.index)
mux.HandleFunc("/upload", a.upload)
mux.HandleFunc("/search", a.search)
+ mux.HandleFunc("/uploads", a.uploads)
}
// index serves the readme on /
-func (a *App) index(w ResponseWriter, r *http.Request) {
+func (a *App) index(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, "static/index.html")
}
diff --git a/storage/app/query.go b/storage/app/query.go
index 9e65397..61b2add 100644
--- a/storage/app/query.go
+++ b/storage/app/query.go
@@ -5,7 +5,9 @@
package app
import (
+ "encoding/json"
"net/http"
+ "strconv"
"golang.org/x/perf/storage/benchfmt"
)
@@ -38,3 +40,49 @@
return
}
}
+
+// uploads serves a list of upload IDs on /uploads.
+// If the query parameter q is provided, only uploads containing matching records are returned.
+// The format of the result is "<count> <uploadid>\n" where count is the number of matching records.
+// The lines are sorted in order from most to least recent.
+// If the query parameter limit is provided, only the most recent limit upload IDs are returned.
+// If limit is not provided, the most recent 1000 upload IDs are returned.
+func (a *App) uploads(w http.ResponseWriter, r *http.Request) {
+ ctx := requestContext(r)
+
+ if err := r.ParseForm(); err != nil {
+ http.Error(w, err.Error(), 500)
+ return
+ }
+
+ q := r.Form.Get("q")
+
+ limit := 1000
+ limitStr := r.Form.Get("limit")
+ if limitStr != "" {
+ var err error
+ limit, err = strconv.Atoi(limitStr)
+ if err != nil {
+ http.Error(w, "invalid limit parameter", 400)
+ return
+ }
+ }
+
+ res := a.DB.ListUploads(q, r.Form["extra_label"], limit)
+ defer res.Close()
+
+ w.Header().Set("Content-Type", "application/json")
+ e := json.NewEncoder(w)
+ for res.Next() {
+ ui := res.Info()
+ if err := e.Encode(&ui); err != nil {
+ errorf(ctx, "failed to encode JSON: %v", err)
+ http.Error(w, err.Error(), 500)
+ return
+ }
+ }
+ if err := res.Err(); err != nil {
+ http.Error(w, err.Error(), 500)
+ return
+ }
+}
diff --git a/storage/app/query_test.go b/storage/app/query_test.go
index c7a7688..15c5274 100644
--- a/storage/app/query_test.go
+++ b/storage/app/query_test.go
@@ -5,12 +5,16 @@
package app
import (
+ "encoding/json"
"fmt"
+ "io"
"mime/multipart"
"net/http"
"net/url"
+ "reflect"
"testing"
+ "golang.org/x/perf/storage"
"golang.org/x/perf/storage/benchfmt"
)
@@ -96,3 +100,87 @@
})
}
}
+
+func TestUploads(t *testing.T) {
+ app := createTestApp(t)
+ defer app.Close()
+
+ // Write 9 uploads to the database. These uploads have 1-9
+ // results each, a common label "i" set to the upload number,
+ // and a label "j" set to the record number within the upload.
+ var uploadIDs []string
+ for i := 0; i < 9; i++ {
+ status := app.uploadFiles(t, func(mpw *multipart.Writer) {
+ w, err := mpw.CreateFormFile("file", "path/1.txt")
+ if err != nil {
+ t.Errorf("CreateFormFile: %v", err)
+ }
+ bp := benchfmt.NewPrinter(w)
+ for j := 0; j <= i; j++ {
+ r := &benchfmt.Result{Labels: map[string]string{"i": fmt.Sprintf("%d", i)}, NameLabels: make(map[string]string), Content: "BenchmarkName 1 ns/op"}
+ r.Labels["j"] = fmt.Sprintf("%d", j)
+ if err := bp.Print(r); err != nil {
+ t.Fatalf("Print: %v", err)
+ }
+ }
+ })
+ uploadIDs = append(uploadIDs, status.UploadID)
+ }
+
+ tests := []struct {
+ q string
+ extraLabels []string
+ want []storage.UploadInfo
+ }{
+ {"", nil, []storage.UploadInfo{
+ {9, uploadIDs[8], nil}, {8, uploadIDs[7], nil}, {7, uploadIDs[6], nil}, {6, uploadIDs[5], nil}, {5, uploadIDs[4], nil}, {4, uploadIDs[3], nil}, {3, uploadIDs[2], nil}, {2, uploadIDs[1], nil}, {1, uploadIDs[0], nil},
+ }},
+ {"j:5", nil, []storage.UploadInfo{{1, uploadIDs[8], nil}, {1, uploadIDs[7], nil}, {1, uploadIDs[6], nil}, {1, uploadIDs[5], nil}}},
+ {"i:5", []string{"i"}, []storage.UploadInfo{{6, uploadIDs[5], benchfmt.Labels{"i": "5"}}}},
+ {"not:found", nil, nil},
+ }
+ for _, test := range tests {
+ t.Run("query="+test.q, func(t *testing.T) {
+ u := app.srv.URL + "/uploads"
+ uv := url.Values{}
+ if test.q != "" {
+ uv["q"] = []string{test.q}
+ }
+ if test.extraLabels != nil {
+ uv["extra_label"] = test.extraLabels
+ }
+ if len(uv) > 0 {
+ u += "?" + uv.Encode()
+ }
+
+ resp, err := http.Get(u)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ t.Fatalf("get /uploads: %v", resp.Status)
+ }
+ dec := json.NewDecoder(resp.Body)
+ i := 0
+ for {
+ var ui storage.UploadInfo
+ if err := dec.Decode(&ui); err == io.EOF {
+ break
+ } else if err != nil {
+ t.Fatalf("failed to parse UploadInfo: %v", err)
+ }
+ if i > len(test.want) {
+ t.Fatalf("too many responses: have %d+ want %d", i, len(test.want))
+ }
+ if !reflect.DeepEqual(ui, test.want[i]) {
+ t.Errorf("uploadinfo = %#v, want %#v", ui, test.want[i])
+ }
+ i++
+ }
+ if i < len(test.want) {
+ t.Fatalf("missing responses: have %d want %d", i, len(test.want))
+ }
+ })
+ }
+}
diff --git a/storage/appengine/static/index.html b/storage/appengine/static/index.html
index 096a293..439f4dd 100644
--- a/storage/appengine/static/index.html
+++ b/storage/appengine/static/index.html
@@ -28,8 +28,8 @@
<p>As a convenience for testing, GET on /upload will render an HTML form that can be used for initiating an upload.</p>
- <h3>GET https://perfdata.golang.org/search?q=$search</h3>
- <p>A GET request to this URL will return a text file with synthesized benchmark results matching the search. The search string contains space-separated "key:value" pairs which limits the results to only records containing those exact fields. Every "key:value" pair is ANDed together, and each value must be matched exactly, with no regexes or substring matches supported. Range queries will be supported for prespecified date fields. Example searches:</p>
+ <h3>GET /search?q=$search</h3>
+ <p>A GET request to this URL will return a text file with synthesized benchmark results matching the search. The search string contains space-separated "key:value" pairs which limits the results to only records containing those exact fields. Every "key:value" pair is ANDed together, and each value must be matched exactly, with no regexes or substring matches supported. The operators ">" and "<" may be used instead of ":" to perform a range query. Example searches:</p>
<ul>
<li>by:rsc pkg:compress/flate commit:1234</li>
@@ -37,5 +37,19 @@
<li>upload:123</li>
<li>commit-time>2016-12-01</li>
</ul>
+
+ <h3>GET /uploads?q=$search&extra_label=$label&limit=$limit</h3>
+ <p>A GET request to this URL returns a list of the most recent <code>$limit</code> uploads that match the search string. If the <code>q</code> parameter is omitted, all uploads will be returned. If the <code>limit</code> parameter is omitted, a server-specified limit is used. If the <code>extra_label</code> parameter is supplied, an arbitrary value for that label will be chosen from the upload's records. (Therefore, this is most useful for labels that do not vary across the upload, such as "by" or "upload-time".)</p>
+ <p>The result of this query is streaming JSON (readable using <a href="https://godoc.org/encoding/json#NewDecoder">>json.NewDecoder</a>), with one JSON entity per upload:</p>
+ <pre>
+{
+ "Count": 10,
+ "UploadID": "arbitrary-string",
+ "LabelValues": {
+ "by": "user@email.com",
+ "upload-time": "2006-01-02T15:04:05Z",
+ }
+}
+ </pre>
</body>
</html>
diff --git a/storage/db/db.go b/storage/db/db.go
index 5db5e0a..6794847 100644
--- a/storage/db/db.go
+++ b/storage/db/db.go
@@ -19,6 +19,7 @@
"unicode"
"golang.org/x/net/context"
+ "golang.org/x/perf/storage"
"golang.org/x/perf/storage/benchfmt"
"golang.org/x/perf/storage/query"
)
@@ -358,6 +359,28 @@
return u.tx.Rollback()
}
+// parseQueryPart parses a single query part into a SQL expression and a list of arguments.
+func parseQueryPart(part string) (sql string, args []interface{}, err error) {
+ sepIndex := strings.IndexFunc(part, func(r rune) bool {
+ return r == ':' || r == '>' || r == '<' || unicode.IsSpace(r) || unicode.IsUpper(r)
+ })
+ if sepIndex < 0 {
+ return "", nil, fmt.Errorf("query part %q is missing operator", part)
+ }
+ key, sep, value := part[:sepIndex], part[sepIndex], part[sepIndex+1:]
+ switch sep {
+ case ':':
+ if value == "" {
+ // TODO(quentin): Implement support for searching for missing labels.
+ return "", nil, fmt.Errorf("missing value for query part %q", part)
+ }
+ return "SELECT UploadID, RecordID FROM RecordLabels WHERE Name = ? AND Value = ?", []interface{}{key, value}, nil
+ case '>', '<':
+ return fmt.Sprintf("SELECT UploadID, RecordID FROM RecordLabels WHERE Name = ? AND Value %c ?", sep), []interface{}{key, value}, nil
+ }
+ return "", nil, fmt.Errorf("query part %q has invalid key", part)
+}
+
// Query searches for results matching the given query string.
//
// The query string is first parsed into quoted words (as in the shell)
@@ -374,27 +397,12 @@
if i > 0 {
query += " INNER JOIN "
}
- sepIndex := strings.IndexFunc(part, func(r rune) bool {
- return r == ':' || r == '>' || r == '<' || unicode.IsSpace(r) || unicode.IsUpper(r)
- })
- if sepIndex < 0 {
- return &Query{err: fmt.Errorf("query part %q is missing operator", part)}
+ partSql, partArgs, err := parseQueryPart(part)
+ if err != nil {
+ return &Query{err: err}
}
- key, sep, value := part[:sepIndex], part[sepIndex], part[sepIndex+1:]
- switch sep {
- case ':':
- if value == "" {
- // TODO(quentin): Implement support for searching for missing labels.
- return &Query{err: fmt.Errorf("missing value for query part %q", part)}
- }
- query += fmt.Sprintf("(SELECT UploadID, RecordID FROM RecordLabels WHERE Name = ? AND Value = ?) t%d", i)
- args = append(args, key, value)
- case '>', '<':
- query += fmt.Sprintf("(SELECT UploadID, RecordID FROM RecordLabels WHERE Name = ? AND Value %c ?) t%d", sep, i)
- args = append(args, key, value)
- default:
- return &Query{err: fmt.Errorf("query part %q has invalid key", part)}
- }
+ query += fmt.Sprintf("(%s) t%d", partSql, i)
+ args = append(args, partArgs...)
if i > 0 {
query += " USING (UploadID, RecordID)"
}
@@ -505,3 +513,118 @@
}
return db.sql.Close()
}
+
+// UploadList is the result of ListUploads.
+// Use Next to advance through the rows, making sure to call Close when done:
+//
+// q := db.ListUploads("key:value")
+// defer q.Close()
+// for q.Next() {
+// info := q.Info()
+// ...
+// }
+// err = q.Err() // get any error encountered during iteration
+// ...
+type UploadList struct {
+ rows *sql.Rows
+ extraLabels []string
+ // from last call to Next
+ count int
+ uploadID string
+ labelValues []sql.NullString
+ err error
+}
+
+// ListUploads searches for uploads containing results matching the given query string.
+// The query may be empty, in which case all uploads will be returned.
+// For each label in extraLabels, one unspecified record's value will be obtained for each upload.
+// If limit is non-zero, only the limit most recent uploads will be returned.
+func (db *DB) ListUploads(q string, extraLabels []string, limit int) *UploadList {
+ qparts := query.SplitWords(q)
+
+ var args []interface{}
+ query := "SELECT j.UploadID, rCount"
+ for i, label := range extraLabels {
+ query += fmt.Sprintf(", (SELECT l%d.Value FROM RecordLabels l%d WHERE l%d.UploadID = j.UploadID AND Name = ? LIMIT 1)", i, i, i)
+ args = append(args, label)
+ }
+ query += " FROM (SELECT UploadID, COUNT(*) as rCount FROM "
+ for i, part := range qparts {
+ if i > 0 {
+ query += " INNER JOIN "
+ }
+ partSql, partArgs, err := parseQueryPart(part)
+ if err != nil {
+ return &UploadList{err: err}
+ }
+ query += fmt.Sprintf("(%s) t%d", partSql, i)
+ args = append(args, partArgs...)
+ if i > 0 {
+ query += " USING (UploadID, RecordID)"
+ }
+ }
+
+ if len(qparts) > 0 {
+ query += " LEFT JOIN"
+ }
+ query += " Records r"
+ if len(qparts) > 0 {
+ query += " USING (UploadID, RecordID)"
+ }
+ query += " GROUP BY UploadID) j LEFT JOIN Uploads u USING (UploadID) ORDER BY u.Day DESC, u.Seq DESC, u.UploadID DESC"
+ if limit != 0 {
+ query += fmt.Sprintf(" LIMIT %d", limit)
+ }
+
+ rows, err := db.sql.Query(query, args...)
+ if err != nil {
+ return &UploadList{err: err}
+ }
+ return &UploadList{rows: rows, extraLabels: extraLabels}
+}
+
+// Next prepares the next result for reading with the Result
+// method. It returns false when there are no more results, either by
+// reaching the end of the input or an error.
+func (ul *UploadList) Next() bool {
+ if ul.err != nil {
+ return false
+ }
+ if !ul.rows.Next() {
+ return false
+ }
+ args := []interface{}{&ul.uploadID, &ul.count}
+ ul.labelValues = make([]sql.NullString, len(ul.extraLabels))
+ for i := range ul.labelValues {
+ args = append(args, &ul.labelValues[i])
+ }
+ ul.err = ul.rows.Scan(args...)
+ if ul.err != nil {
+ return false
+ }
+ return ul.err == nil
+}
+
+// Info returns the most recent UploadInfo generated by a call to Next.
+func (ul *UploadList) Info() storage.UploadInfo {
+ l := make(benchfmt.Labels)
+ for i := range ul.extraLabels {
+ if ul.labelValues[i].Valid {
+ l[ul.extraLabels[i]] = ul.labelValues[i].String
+ }
+ }
+ return storage.UploadInfo{UploadID: ul.uploadID, Count: ul.count, LabelValues: l}
+}
+
+// Err returns the error state of the query.
+func (ul *UploadList) Err() error {
+ return ul.err
+}
+
+// Close frees resources associated with the query.
+func (ul *UploadList) Close() error {
+ if ul.rows != nil {
+ return ul.rows.Close()
+ }
+ return ul.err
+}
diff --git a/storage/db/db_test.go b/storage/db/db_test.go
index 97c9382..b716623 100644
--- a/storage/db/db_test.go
+++ b/storage/db/db_test.go
@@ -11,6 +11,8 @@
"io/ioutil"
"os"
"os/exec"
+ "reflect"
+ "strconv"
"strings"
"testing"
"time"
@@ -342,3 +344,88 @@
return string(data)
}
+
+// TestListUploads verifies that ListUploads returns the correct values.
+func TestListUploads(t *testing.T) {
+ SetNow(time.Unix(0, 0))
+ defer SetNow(time.Time{})
+ db, cleanup := dbtest.NewDB(t)
+ defer cleanup()
+
+ for i := 0; i < 9; i++ {
+ u, err := db.NewUpload(context.Background())
+ if err != nil {
+ t.Fatalf("NewUpload: %v", err)
+ }
+ for j := 0; j <= i; j++ {
+ labels := benchfmt.Labels{
+ "key": "value",
+ "i": fmt.Sprintf("%d", i),
+ "j": fmt.Sprintf("%d", j),
+ }
+ if err := u.InsertRecord(&benchfmt.Result{
+ labels,
+ nil,
+ 1,
+ fmt.Sprintf("BenchmarkName %d ns/op", j),
+ }); err != nil {
+ t.Fatalf("InsertRecord: %v", err)
+ }
+ }
+ if err := u.Commit(); err != nil {
+ t.Fatalf("Commit: %v", err)
+ }
+ }
+
+ type result struct {
+ count int
+ id string
+ }
+
+ tests := []struct {
+ query string
+ extraLabels []string
+ limit int
+ want []result
+ }{
+ {"", nil, 0, []result{{9, "19700101.9"}, {8, "19700101.8"}, {7, "19700101.7"}, {6, "19700101.6"}, {5, "19700101.5"}, {4, "19700101.4"}, {3, "19700101.3"}, {2, "19700101.2"}, {1, "19700101.1"}}},
+ {"", nil, 2, []result{{9, "19700101.9"}, {8, "19700101.8"}}},
+ {"j:5", nil, 0, []result{{1, "19700101.9"}, {1, "19700101.8"}, {1, "19700101.7"}, {1, "19700101.6"}}},
+ {"i:5", nil, 0, []result{{6, "19700101.6"}}},
+ {"i:5", []string{"i", "missing"}, 0, []result{{6, "19700101.6"}}},
+ {"not:found", nil, 0, nil},
+ }
+
+ for _, test := range tests {
+ t.Run(fmt.Sprintf("query=%s/limit=%d", test.query, test.limit), func(t *testing.T) {
+ r := db.ListUploads(test.query, test.extraLabels, test.limit)
+ defer r.Close()
+ var have []result
+ for r.Next() {
+ ui := r.Info()
+ res := result{ui.Count, ui.UploadID}
+ have = append(have, res)
+ for k, v := range ui.LabelValues {
+ switch k {
+ case "i":
+ uploadNum, err := strconv.Atoi(res.id[strings.LastIndex(res.id, ".")+1:])
+ if err != nil {
+ t.Fatalf("cannot parse upload ID %q", res.id)
+ }
+ if v != fmt.Sprintf("%d", uploadNum-1) {
+ t.Errorf(`i = %q, want "%d"`, v, uploadNum-1)
+ }
+ default:
+ t.Errorf("unexpected label %q", k)
+ }
+ }
+ }
+ if err := r.Err(); err != nil {
+ t.Errorf("Err() = %v", err)
+ }
+ if !reflect.DeepEqual(have, test.want) {
+ t.Errorf("results = %v, want %v", have, test.want)
+ }
+ })
+ }
+}
