internal/govulncheck: copy from x/vuln repo
Currently, gopls maintains an edited copy of the govulncheck command
logic in the internal/vulncheck directory, along with the necessary
glue to make it work with gopls.
This CL is the first in a sequence that will make it easier for gopls
to use that logic.
It creates a new package, internal/govulncheck, adds a script to copy
the corresponding package from the x/vuln repo, and removes the cache
in internal/vulncheck in favor of the copied one.
Although it might appear simpler to copy the .go files directly into
internal/vulncheck, that would require editing the package directives
in the files, and has the risk of overwriting files.
Change-Id: I00f726f7b142048da2407f212873420df54844b3
Reviewed-on: https://go-review.googlesource.com/c/tools/+/405997
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Hyang-Ah Hana Kim <hyangah@gmail.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
gopls-CI: kokoro <noreply+kokoro@google.com>
diff --git a/gopls/internal/govulncheck/README.md b/gopls/internal/govulncheck/README.md
new file mode 100644
index 0000000..d8339c5
--- /dev/null
+++ b/gopls/internal/govulncheck/README.md
@@ -0,0 +1,17 @@
+# internal/govulncheck package
+
+This package is a literal copy of the cmd/govulncheck/internal/govulncheck
+package in the vuln repo (https://go.googlesource.com/vuln).
+
+The `copy.sh` does the copying, after removing all .go files here. To use it:
+
+1. Clone the vuln repo to a directory next to the directory holding this repo
+ (tools). After doing that your directory structure should look something like
+ ```
+ ~/repos/x/tools/gopls/...
+ ~/repos/x/vuln/...
+ ```
+
+2. cd to this directory.
+
+3. Run `copy.sh`.
diff --git a/gopls/internal/vulncheck/cache.go b/gopls/internal/govulncheck/cache.go
similarity index 82%
rename from gopls/internal/vulncheck/cache.go
rename to gopls/internal/govulncheck/cache.go
index 39a38fb..404c356 100644
--- a/gopls/internal/vulncheck/cache.go
+++ b/gopls/internal/govulncheck/cache.go
@@ -2,7 +2,11 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-package vulncheck
+//go:build go1.18
+// +build go1.18
+
+// Package govulncheck supports the govulncheck command.
+package govulncheck
import (
"encoding/json"
@@ -17,8 +21,6 @@
"golang.org/x/vuln/osv"
)
-// copy from x/vuln/cmd/govulncheck/cache.go
-
// The cache uses a single JSON index file for each vulnerability database
// which contains the map from packages to the time the last
// vulnerability for that package was added/modified and the time that
@@ -37,19 +39,22 @@
// $GOPATH/pkg/mod/cache/download/vulndb/{db hostname}/{import path}/vulns.json
// []*osv.Entry
-// fsCache is a thread-safe file-system cache implementing osv.Cache
+// FSCache is a thread-safe file-system cache implementing osv.Cache
//
// TODO: use something like cmd/go/internal/lockedfile for thread safety?
-type fsCache struct {
+type FSCache struct {
mu sync.Mutex
rootDir string
}
+// Assert that *FSCache implements client.Cache.
+var _ client.Cache = (*FSCache)(nil)
+
// use cfg.GOMODCACHE available in cmd/go/internal?
var defaultCacheRoot = filepath.Join(build.Default.GOPATH, "/pkg/mod/cache/download/vulndb")
-func defaultCache() *fsCache {
- return &fsCache{rootDir: defaultCacheRoot}
+func DefaultCache() *FSCache {
+ return &FSCache{rootDir: defaultCacheRoot}
}
type cachedIndex struct {
@@ -57,7 +62,7 @@
Index client.DBIndex
}
-func (c *fsCache) ReadIndex(dbName string) (client.DBIndex, time.Time, error) {
+func (c *FSCache) ReadIndex(dbName string) (client.DBIndex, time.Time, error) {
c.mu.Lock()
defer c.mu.Unlock()
@@ -75,7 +80,7 @@
return index.Index, index.Retrieved, nil
}
-func (c *fsCache) WriteIndex(dbName string, index client.DBIndex, retrieved time.Time) error {
+func (c *FSCache) WriteIndex(dbName string, index client.DBIndex, retrieved time.Time) error {
c.mu.Lock()
defer c.mu.Unlock()
@@ -96,7 +101,7 @@
return nil
}
-func (c *fsCache) ReadEntries(dbName string, p string) ([]*osv.Entry, error) {
+func (c *FSCache) ReadEntries(dbName string, p string) ([]*osv.Entry, error) {
c.mu.Lock()
defer c.mu.Unlock()
@@ -114,7 +119,7 @@
return entries, nil
}
-func (c *fsCache) WriteEntries(dbName string, p string, entries []*osv.Entry) error {
+func (c *FSCache) WriteEntries(dbName string, p string, entries []*osv.Entry) error {
c.mu.Lock()
defer c.mu.Unlock()
diff --git a/gopls/internal/govulncheck/cache_test.go b/gopls/internal/govulncheck/cache_test.go
new file mode 100644
index 0000000..5a25c78
--- /dev/null
+++ b/gopls/internal/govulncheck/cache_test.go
@@ -0,0 +1,165 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.18
+// +build go1.18
+
+package govulncheck
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "reflect"
+ "testing"
+ "time"
+
+ "golang.org/x/sync/errgroup"
+ "golang.org/x/vuln/client"
+ "golang.org/x/vuln/osv"
+)
+
+func TestCache(t *testing.T) {
+ tmpDir := t.TempDir()
+
+ cache := &FSCache{rootDir: tmpDir}
+ dbName := "vulndb.golang.org"
+
+ _, _, err := cache.ReadIndex(dbName)
+ if err != nil {
+ t.Fatalf("ReadIndex failed for non-existent database: %v", err)
+ }
+
+ if err = os.Mkdir(filepath.Join(tmpDir, dbName), 0777); err != nil {
+ t.Fatalf("os.Mkdir failed: %v", err)
+ }
+ _, _, err = cache.ReadIndex(dbName)
+ if err != nil {
+ t.Fatalf("ReadIndex failed for database without cached index: %v", err)
+ }
+
+ now := time.Now()
+ expectedIdx := client.DBIndex{
+ "a.vuln.example.com": time.Time{}.Add(time.Hour),
+ "b.vuln.example.com": time.Time{}.Add(time.Hour * 2),
+ "c.vuln.example.com": time.Time{}.Add(time.Hour * 3),
+ }
+ if err = cache.WriteIndex(dbName, expectedIdx, now); err != nil {
+ t.Fatalf("WriteIndex failed to write index: %v", err)
+ }
+
+ idx, retrieved, err := cache.ReadIndex(dbName)
+ if err != nil {
+ t.Fatalf("ReadIndex failed for database with cached index: %v", err)
+ }
+ if !reflect.DeepEqual(idx, expectedIdx) {
+ t.Errorf("ReadIndex returned unexpected index, got:\n%s\nwant:\n%s", idx, expectedIdx)
+ }
+ if !retrieved.Equal(now) {
+ t.Errorf("ReadIndex returned unexpected retrieved: got %s, want %s", retrieved, now)
+ }
+
+ if _, err = cache.ReadEntries(dbName, "vuln.example.com"); err != nil {
+ t.Fatalf("ReadEntires failed for non-existent package: %v", err)
+ }
+
+ expectedEntries := []*osv.Entry{
+ {ID: "001"},
+ {ID: "002"},
+ {ID: "003"},
+ }
+ if err := cache.WriteEntries(dbName, "vuln.example.com", expectedEntries); err != nil {
+ t.Fatalf("WriteEntries failed: %v", err)
+ }
+
+ entries, err := cache.ReadEntries(dbName, "vuln.example.com")
+ if err != nil {
+ t.Fatalf("ReadEntries failed for cached package: %v", err)
+ }
+ if !reflect.DeepEqual(entries, expectedEntries) {
+ t.Errorf("ReadEntries returned unexpected entries, got:\n%v\nwant:\n%v", entries, expectedEntries)
+ }
+}
+
+func TestConcurrency(t *testing.T) {
+ tmpDir := t.TempDir()
+
+ cache := &FSCache{rootDir: tmpDir}
+ dbName := "vulndb.golang.org"
+
+ g := new(errgroup.Group)
+ for i := 0; i < 1000; i++ {
+ i := i
+ g.Go(func() error {
+ id := i % 5
+ p := fmt.Sprintf("package%d", id)
+
+ entries, err := cache.ReadEntries(dbName, p)
+ if err != nil {
+ return err
+ }
+
+ err = cache.WriteEntries(dbName, p, append(entries, &osv.Entry{ID: fmt.Sprint(id)}))
+ if err != nil {
+ return err
+ }
+ return nil
+ })
+ }
+
+ if err := g.Wait(); err != nil {
+ t.Errorf("error in parallel cache entries read/write: %v", err)
+ }
+
+ // sanity checking
+ for i := 0; i < 5; i++ {
+ id := fmt.Sprint(i)
+ p := fmt.Sprintf("package%s", id)
+
+ es, err := cache.ReadEntries(dbName, p)
+ if err != nil {
+ t.Fatalf("failed to read entries: %v", err)
+ }
+ for _, e := range es {
+ if e.ID != id {
+ t.Errorf("want %s ID for vuln entry; got %s", id, e.ID)
+ }
+ }
+ }
+
+ // do similar for cache index
+ start := time.Now()
+ for i := 0; i < 1000; i++ {
+ i := i
+ g.Go(func() error {
+ id := i % 5
+ p := fmt.Sprintf("package%v", id)
+
+ idx, _, err := cache.ReadIndex(dbName)
+ if err != nil {
+ return err
+ }
+
+ if idx == nil {
+ idx = client.DBIndex{}
+ }
+
+ // sanity checking
+ if rt, ok := idx[p]; ok && rt.Before(start) {
+ return fmt.Errorf("unexpected past time in index: %v before start %v", rt, start)
+ }
+
+ now := time.Now()
+ idx[p] = now
+ if err := cache.WriteIndex(dbName, idx, now); err != nil {
+ return err
+ }
+ return nil
+ })
+ }
+
+ if err := g.Wait(); err != nil {
+ t.Errorf("error in parallel cache index read/write: %v", err)
+ }
+}
diff --git a/gopls/internal/govulncheck/copy.sh b/gopls/internal/govulncheck/copy.sh
new file mode 100755
index 0000000..24ed45b
--- /dev/null
+++ b/gopls/internal/govulncheck/copy.sh
@@ -0,0 +1,13 @@
+#!/bin/bash -eu
+
+# Copyright 2020 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+set -o pipefail
+
+# Copy golang.org/x/vuln/cmd/govulncheck/internal/govulncheck into this directory.
+# Assume the x/vuln repo is a sibling of the tools repo.
+
+rm -f *.go
+cp ../../../../vuln/cmd/govulncheck/internal/govulncheck/*.go .
diff --git a/gopls/internal/vulncheck/command.go b/gopls/internal/vulncheck/command.go
index 3fd9d03..459ecca 100644
--- a/gopls/internal/vulncheck/command.go
+++ b/gopls/internal/vulncheck/command.go
@@ -14,6 +14,7 @@
"strings"
"golang.org/x/tools/go/packages"
+ gvc "golang.org/x/tools/gopls/internal/govulncheck"
"golang.org/x/tools/internal/lsp/command"
"golang.org/x/vuln/client"
"golang.org/x/vuln/vulncheck"
@@ -28,7 +29,7 @@
args.Pattern = "."
}
- dbClient, err := client.NewClient(findGOVULNDB(cfg), client.Options{HTTPCache: defaultCache()})
+ dbClient, err := client.NewClient(findGOVULNDB(cfg), client.Options{HTTPCache: gvc.DefaultCache()})
if err != nil {
return res, err
}
