blob: 7f056c4ed9fbd3280f3446c2c76d0b6c4874a567 [file] [log] [blame]
Jonathan Amsterdamf78f1452022-06-09 12:38:39 -04001// Copyright 2022 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// Only run this on Go 1.18 or higher, because govulncheck can't
6// run on binaries before 1.18.
7
8//go:build go1.18
9// +build go1.18
10
11package main
12
13import (
Ian Cottrelldc4d5012023-04-23 16:23:07 -040014 "bytes"
15 "context"
Jonathan Amsterdamf78f1452022-06-09 12:38:39 -040016 "flag"
17 "fmt"
18 "os"
Jonathan Amsterdamf78f1452022-06-09 12:38:39 -040019 "path/filepath"
20 "regexp"
Zvonimir Pavlinovic4782cc12023-04-17 10:33:37 -070021 "runtime"
Ian Cottrellcc61b4c2023-05-15 20:06:41 -040022 "sync"
Jonathan Amsterdamf78f1452022-06-09 12:38:39 -040023 "testing"
Bryan C. Millse55aa3a2023-04-26 16:36:56 -040024 "unsafe"
Jonathan Amsterdamf78f1452022-06-09 12:38:39 -040025
26 "github.com/google/go-cmdtest"
Ian Cottrell7b0106d2023-06-05 22:42:20 -040027 "golang.org/x/vuln/internal/govulncheck"
Julie Qiu2195bd12023-03-07 22:09:32 -050028 "golang.org/x/vuln/internal/test"
Hana (Hyang-Ah) Kim2aa05532022-10-03 13:20:49 -040029 "golang.org/x/vuln/internal/web"
Julie Qiu62092d52023-05-03 10:26:25 -040030 "golang.org/x/vuln/scan"
Jonathan Amsterdamf78f1452022-06-09 12:38:39 -040031)
32
33var update = flag.Bool("update", false, "update test files with results")
34
Ian Cottrellbb1cd572023-05-15 16:06:54 -040035type fixup struct {
36 pattern string
37 compiled *regexp.Regexp
38 replace string
39 replaceFunc func(b []byte) []byte
40}
41
42var fixups = []fixup{
43 {
44 // modifies paths to Go files by replacing their directory with "...".
45 // For example,/a/b/c.go becomes .../c.go .
46 // This makes it possible to compare govulncheck output across systems, because
47 // Go filenames include setup-specific paths.
48 pattern: `[^\s"]*\.go[\s":]`,
49 replaceFunc: func(b []byte) []byte {
50 s := string(b)
51 return []byte(fmt.Sprintf(`.../%s%c`, filepath.Base(s[:len(s)-1]), s[len(s)-1]))
52 },
53 }, {
54 // There was a one-line change in container/heap/heap.go between 1.18
55 // and 1.19 that makes the stack traces different. Ignore it.
56 pattern: `heap\.go:(\d+)`,
57 replace: `N`,
58 }, {
59 pattern: `Scanning your code and (\d+) packages across (\d+)`,
60 replace: `Scanning your code and P packages across M`,
61 }, {
Ian Cottrelle3a5c492023-08-15 17:52:15 -040062 pattern: `Scanner: govulncheck@v.*`,
63 replace: `Scanner: govulncheck@v1.0.0`,
Ian Cottrellbb1cd572023-05-15 16:06:54 -040064 }, {
65 pattern: `"([^"]*") is a file`,
66 replace: `govulncheck: myfile is a file`,
67 }, {
68 pattern: `"scanner_version": "[^"]*"`,
69 replace: `"scanner_version": "v0.0.0-00000000000-20000101010101"`,
70 }, {
71 pattern: `file:///(.*)/testdata/vulndb`,
72 replace: `testdata/vulndb`,
73 }, {
74 pattern: `package (.*) is not in (GOROOT|std) (.*)`,
75 replace: `package foo is not in GOROOT (/tmp/foo)`,
76 }, {
77 pattern: `modified (.*)\)`,
78 replace: `modified 01 Jan 21 00:00 UTC)`,
79 }, {
Ian Cottrelle3a5c492023-08-15 17:52:15 -040080 pattern: `Go: (go1.[\.\d]*|devel).*`,
81 replace: `Go: go1.18`,
Ian Cottrellbb1cd572023-05-15 16:06:54 -040082 }, {
83 pattern: `"go_version": "go[^\s"]*"`,
84 replace: `"go_version": "go1.18"`,
85 },
86}
87
88func (f *fixup) init() {
89 f.compiled = regexp.MustCompile(f.pattern)
90}
91
92func (f *fixup) apply(data []byte) []byte {
93 if f.replaceFunc != nil {
94 return f.compiled.ReplaceAllFunc(data, f.replaceFunc)
95 }
96 return f.compiled.ReplaceAll(data, []byte(f.replace))
97}
98
99func init() {
100 for i := range fixups {
101 fixups[i].init()
102 }
103}
104
Jonathan Amsterdamf78f1452022-06-09 12:38:39 -0400105func TestCommand(t *testing.T) {
Dmitri Shuralyov1568f332023-07-18 14:17:03 -0400106 if testing.Short() {
107 t.Skip("skipping test that uses internet in short mode")
108 }
109
Jonathan Amsterdamf78f1452022-06-09 12:38:39 -0400110 testDir, err := os.Getwd()
111 if err != nil {
112 t.Fatal(err)
113 }
Zvonimir Pavlinovic4782cc12023-04-17 10:33:37 -0700114 vulndbDir, err := filepath.Abs(filepath.Join(testDir, "testdata", "vulndb-v1"))
115 if err != nil {
116 t.Fatal(err)
117 }
Ian Cottrell06356f82023-05-15 21:51:36 -0400118 govulndbURI, err := web.URLFromFilePath(vulndbDir)
119 if err != nil {
120 t.Fatalf("failed to create make vulndb url: %v", err)
121 }
Jonathan Amsterdamf78f1452022-06-09 12:38:39 -0400122
Jonathan Amsterdamf78f1452022-06-09 12:38:39 -0400123 moduleDirs, err := filepath.Glob("testdata/modules/*")
124 if err != nil {
125 t.Fatal(err)
126 }
Zvonimir Pavlinovicbf90c092022-08-23 11:21:49 -0700127
Jonathan Amsterdamcf2b5652022-09-21 08:14:20 -0400128 os.Setenv("moddir", filepath.Join(testDir, "testdata", "modules"))
Jonathan Amsterdamf78f1452022-06-09 12:38:39 -0400129 for _, md := range moduleDirs {
Zvonimir Pavlinovic9a72d682023-01-18 17:24:09 -0800130 // Skip nogomod module. It has intended build issues.
131 if filepath.Base(md) == "nogomod" {
Zvonimir Pavlinovicbf90c092022-08-23 11:21:49 -0700132 continue
133 }
Julie Qiu22f32fd2023-04-18 17:52:36 -0400134
135 // Build test module binary.
Ian Cottrellb57773a2023-05-15 20:15:18 -0400136 binary, cleanup := test.GoBuild(t, md, "", filepath.Base(md) == "strip")
Jonathan Amsterdamcf2b5652022-09-21 08:14:20 -0400137 t.Cleanup(cleanup)
Jonathan Amsterdamf78f1452022-06-09 12:38:39 -0400138 // Set an environment variable to the path to the binary, so tests
139 // can refer to it.
140 varName := filepath.Base(md) + "_binary"
141 os.Setenv(varName, binary)
142 }
Ian Cottrell78e1e632023-05-15 22:33:29 -0400143 runTestSuite(t, filepath.Join(testDir, "testdata"), govulndbURI.String(), *update)
Ian Cottrellb57773a2023-05-15 20:15:18 -0400144 if runtime.GOOS != "darwin" {
Zvonimir Pavlinovic2f478822023-04-19 15:10:36 -0700145 // TODO(https://go.dev/issue/61051): binaries are not currently stripped on darwin.
146 // This is expected to change in Go 1.22.
Ian Cottrell78e1e632023-05-15 22:33:29 -0400147 runTestSuite(t, filepath.Join(testDir, "testdata/strip"), govulndbURI.String(), *update)
Zvonimir Pavlinovic4782cc12023-04-17 10:33:37 -0700148 }
149}
150
Ian Cottrellcc61b4c2023-05-15 20:06:41 -0400151// Limit the number of concurrent scans. Scanning is implemented using
152// x/tools/go/ssa, which is known to be memory-hungry
153// (see https://go.dev/issue/14113), and by default the testing package
154// allows up to GOMAXPROCS parallel tests at a time.
155//
156// For now we arbitrarily limit to ⌈GOMAXPROCS/4⌉, on the theory that many Go
157// developer and CI machines have at least 8 logical cores and we want most
158// runs of the test to exercise at least a little concurrency. If that turns
159// out to still be too high, we may consider reducing it further.
160//
161// Since all of the scans run in the same process, we need an especially low
162// limit on 32-bit platforms: we may run out of virtual address space well
163// before we run out of system RAM.
164var (
165 parallelLimiter chan struct{}
166 parallelLimiterInit sync.Once
167)
168
Zvonimir Pavlinovic4782cc12023-04-17 10:33:37 -0700169// testSuite creates a cmdtest suite from dir. It also defines
170// a govulncheck command on the suite that runs govulncheck
171// against vulnerability database available at vulndbDir.
Ian Cottrell06356f82023-05-15 21:51:36 -0400172func runTestSuite(t *testing.T, dir string, govulndb string, update bool) {
Ian Cottrellb57773a2023-05-15 20:15:18 -0400173 parallelLimiterInit.Do(func() {
174 limit := (runtime.GOMAXPROCS(0) + 3) / 4
175 if limit > 2 && unsafe.Sizeof(uintptr(0)) < 8 {
176 limit = 2
177 }
178 parallelLimiter = make(chan struct{}, limit)
179 })
180
Zvonimir Pavlinovic4782cc12023-04-17 10:33:37 -0700181 ts, err := cmdtest.Read(dir)
182 if err != nil {
Ian Cottrellb57773a2023-05-15 20:15:18 -0400183 t.Fatal(err)
Zvonimir Pavlinovic4782cc12023-04-17 10:33:37 -0700184 }
Ian Cottrelld8818052023-04-21 22:20:55 -0400185 ts.DisableLogging = true
Bryan C. Millse55aa3a2023-04-26 16:36:56 -0400186
Zvonimir Pavlinovic4782cc12023-04-17 10:33:37 -0700187 ts.Commands["govulncheck"] = func(args []string, inputFile string) ([]byte, error) {
Ian Cottrellcc61b4c2023-05-15 20:06:41 -0400188 parallelLimiter <- struct{}{}
189 defer func() { <-parallelLimiter }()
Bryan C. Millse55aa3a2023-04-26 16:36:56 -0400190
Ian Cottrell06356f82023-05-15 21:51:36 -0400191 newargs := append([]string{"-db", govulndb}, args...)
Zvonimir Pavlinovic4782cc12023-04-17 10:33:37 -0700192
Ian Cottrelldc4d5012023-04-23 16:23:07 -0400193 buf := &bytes.Buffer{}
194 cmd := scan.Command(context.Background(), newargs...)
195 cmd.Stdout = buf
196 cmd.Stderr = buf
Zvonimir Pavlinovic4782cc12023-04-17 10:33:37 -0700197 if inputFile != "" {
Ian Cottrell78e1e632023-05-15 22:33:29 -0400198 input, err := os.Open(filepath.Join(dir, inputFile))
199 if err != nil {
200 return nil, err
201 }
202 defer input.Close()
203 cmd.Stdin = input
Zvonimir Pavlinovic4782cc12023-04-17 10:33:37 -0700204 }
205 // We set GOVERSION to always get the same results regardless of the underlying Go build system.
Ian Cottrell9e5885c2023-05-26 19:38:17 -0400206 cmd.Env = append(os.Environ(), "GOVERSION=go1.18")
Ian Cottrella2fb4c42023-05-11 19:29:44 -0400207 if err := cmd.Start(); err != nil {
208 return nil, err
209 }
Ian Cottrell06356f82023-05-15 21:51:36 -0400210 err := cmd.Wait()
Ian Cottrelldc4d5012023-04-23 16:23:07 -0400211 switch e := err.(type) {
212 case nil:
213 case interface{ ExitCode() int }:
214 err = &cmdtest.ExitCodeErr{Msg: err.Error(), Code: e.ExitCode()}
215 if e.ExitCode() == 0 {
216 err = nil
217 }
218 default:
219 fmt.Fprintln(buf, err)
220 err = &cmdtest.ExitCodeErr{Msg: err.Error(), Code: 1}
221 }
Ian Cottrell7b0106d2023-06-05 22:42:20 -0400222 sorted := buf
223 if err == nil && isJSONMode(args) {
224 // parse, sort and reprint the output for test stability
225 gather := test.NewMockHandler()
226 if err := govulncheck.HandleJSON(buf, gather); err != nil {
227 return nil, err
228 }
229 sorted = &bytes.Buffer{}
230 h := govulncheck.NewJSONHandler(sorted)
231 if err := gather.Write(h); err != nil {
232 return nil, err
233 }
234 }
235 out := sorted.Bytes()
Ian Cottrellbb1cd572023-05-15 16:06:54 -0400236 for _, fix := range fixups {
237 out = fix.apply(out)
238 }
Zvonimir Pavlinovic4782cc12023-04-17 10:33:37 -0700239 return out, err
240 }
Ian Cottrellf97f4a92023-05-19 18:10:24 -0400241 if update {
242 ts.Run(t, true)
243 return
244 }
245 ts.RunParallel(t, false)
Zvonimir Pavlinovic4782cc12023-04-17 10:33:37 -0700246}
Ian Cottrell7b0106d2023-06-05 22:42:20 -0400247
248func isJSONMode(args []string) bool {
249 for _, arg := range args {
250 if arg == "-json" {
251 return true
252 }
253 }
254 return false
255}