internal/vulndbclient: add logic for fetching all vulnerabilities
Change-Id: I45df7e8d9abe8f2f9b4e57c198f1a4129e8a8837
Reviewed-on: https://go-review.googlesource.com/c/pkgsite-metrics/+/488175
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Maceo Thompson <maceothompson@google.com>
Run-TryBot: Zvonimir Pavlinovic <zpavlinovic@google.com>
diff --git a/internal/worker/vulndb.go b/internal/worker/vulndb.go
new file mode 100644
index 0000000..f681c6d
--- /dev/null
+++ b/internal/worker/vulndb.go
@@ -0,0 +1,64 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package worker
+
+import (
+ "context"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "cloud.google.com/go/storage"
+ "golang.org/x/vuln/osv"
+ "google.golang.org/api/iterator"
+)
+
+// gcsOSVPrefix is the directory under which .json
+// files with OSV entries are located.
+const gcsOSVPrefix = "ID"
+
+// allVulnerabilities fetches all osv.Entries from GCS bucket located at ID/*.json paths.
+func allVulnerabilities(ctx context.Context, bucket *storage.BucketHandle) ([]*osv.Entry, error) {
+ var entries []*osv.Entry
+ query := &storage.Query{Prefix: gcsOSVPrefix}
+ it := bucket.Objects(ctx, query)
+ for {
+ attrs, err := it.Next()
+ if err == iterator.Done {
+ break
+ }
+ if err != nil {
+ return nil, err
+ }
+ // Skip zip files and index.json.
+ if !strings.HasSuffix(attrs.Name, ".json") || strings.HasSuffix(attrs.Name, "index.json") {
+ continue
+ }
+
+ e, err := readEntry(ctx, bucket, attrs.Name)
+ if err != nil {
+ return nil, err
+ }
+ entries = append(entries, e)
+ }
+ return entries, nil
+}
+
+func readEntry(ctx context.Context, bucket *storage.BucketHandle, gcsPath string) (*osv.Entry, error) {
+ localPath := filepath.Join(os.TempDir(), "binary")
+ if err := copyToLocalFile(localPath, false, gcsPath, gcsOpenFileFunc(ctx, bucket)); err != nil {
+ return nil, err
+ }
+ js, err := os.ReadFile(localPath)
+ if err != nil {
+ return nil, err
+ }
+ var entry osv.Entry
+ if err := json.Unmarshal(js, &entry); err != nil {
+ return nil, err
+ }
+ return &entry, nil
+}
diff --git a/internal/worker/vulndb_test.go b/internal/worker/vulndb_test.go
new file mode 100644
index 0000000..c6c2b3f
--- /dev/null
+++ b/internal/worker/vulndb_test.go
@@ -0,0 +1,34 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package worker
+
+import (
+ "context"
+ "testing"
+
+ "cloud.google.com/go/storage"
+ test "golang.org/x/pkgsite-metrics/internal/testing"
+)
+
+func TestIntegrationAllVulns(t *testing.T) {
+ test.NeedsIntegrationEnv(t)
+
+ ctx := context.Background()
+ c, err := storage.NewClient(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ bucket := c.Bucket("go-vulndb")
+ if bucket == nil {
+ t.Fatal("failed to create go-vulndb bucket")
+ }
+ es, err := allVulnerabilities(ctx, bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(es) == 0 {
+ t.Fatal("want some vulnerabilities; got none")
+ }
+}
