x/playground: allow playground to run fuzz tests
Current implementation does not check for Fuzz functions. In case we
have test functions and no main, this change moves a file as
prog_test.go and runs go test -c.
Change also includes TestMain function, and removeBanner
Fixes golang/go#57029
Fixes golang/go#32237
Fixes golang/go#45431
Change-Id: I249df00a98db484a04a1380c73e2ff268a2e3379
Reviewed-on: https://go-review.googlesource.com/c/playground/+/456355
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Jenny Rakoczy <jenny@golang.org>
Run-TryBot: Bryan Mills <bcmills@google.com>
Reviewed-by: Bryan Mills <bcmills@google.com>
Reviewed-by: Cherry Mui <cherryyz@google.com>
diff --git a/sandbox.go b/sandbox.go
index 4854cc5..70eaa95 100644
--- a/sandbox.go
+++ b/sandbox.go
@@ -28,7 +28,6 @@
"strconv"
"strings"
"sync"
- "text/template"
"time"
"unicode"
"unicode/utf8"
@@ -49,7 +48,8 @@
// progName is the implicit program name written to the temp
// dir and used in compiler and vet errors.
- progName = "prog.go"
+ progName = "prog.go"
+ progTestName = "prog_test.go"
)
const (
@@ -171,7 +171,7 @@
return fmt.Sprintf("%s-%s-%x", prefix, runtime.Version(), h.Sum(nil))
}
-// isTestFunc tells whether fn has the type of a testing function.
+// isTestFunc tells whether fn has the type of a testing, or fuzz function, or a TestMain func.
func isTestFunc(fn *ast.FuncDecl) bool {
if fn.Type.Results != nil && len(fn.Type.Results.List) > 0 ||
fn.Type.Params.List == nil ||
@@ -183,19 +183,19 @@
if !ok {
return false
}
- // We can't easily check that the type is *testing.T
+ // We can't easily check that the type is *testing.T or *testing.F
// because we don't know how testing has been imported,
- // but at least check that it's *T or *something.T.
- if name, ok := ptr.X.(*ast.Ident); ok && name.Name == "T" {
+ // but at least check that it's *T (or *F) or *something.T (or *something.F).
+ if name, ok := ptr.X.(*ast.Ident); ok && (name.Name == "T" || name.Name == "F" || name.Name == "M") {
return true
}
- if sel, ok := ptr.X.(*ast.SelectorExpr); ok && sel.Sel.Name == "T" {
+ if sel, ok := ptr.X.(*ast.SelectorExpr); ok && (sel.Sel.Name == "T" || sel.Sel.Name == "F" || sel.Sel.Name == "M") {
return true
}
return false
}
-// isTest tells whether name looks like a test (or benchmark, according to prefix).
+// isTest tells whether name looks like a test (or benchmark, or fuzz, according to prefix).
// It is a Test (say) if there is a character after Test that is not a lower-case letter.
// We don't want mistaken Testimony or erroneous Benchmarking.
func isTest(name, prefix string) bool {
@@ -213,32 +213,22 @@
// If the main function is present or there are no tests or examples, it returns nil.
// getTestProg emulates the "go test" command as closely as possible.
// Benchmarks are not supported because of sandboxing.
-func getTestProg(src []byte) []byte {
+func isTestProg(src []byte) bool {
fset := token.NewFileSet()
// Early bail for most cases.
f, err := parser.ParseFile(fset, progName, src, parser.ImportsOnly)
if err != nil || f.Name.Name != "main" {
- return nil
- }
-
- // importPos stores the position to inject the "testing" import declaration, if needed.
- importPos := fset.Position(f.Name.End()).Offset
-
- var testingImported bool
- for _, s := range f.Imports {
- if s.Path.Value == `"testing"` && s.Name == nil {
- testingImported = true
- break
- }
+ return false
}
// Parse everything and extract test names.
f, err = parser.ParseFile(fset, progName, src, parser.ParseComments)
if err != nil {
- return nil
+ return false
}
- var tests []string
+ var hasTest bool
+ var hasFuzz bool
for _, d := range f.Decls {
n, ok := d.(*ast.FuncDecl)
if !ok {
@@ -249,79 +239,24 @@
case name == "main":
// main declared as a method will not obstruct creation of our main function.
if n.Recv == nil {
- return nil
+ return false
}
+ case name == "TestMain" && isTestFunc(n):
+ hasTest = true
case isTest(name, "Test") && isTestFunc(n):
- tests = append(tests, name)
+ hasTest = true
+ case isTest(name, "Fuzz") && isTestFunc(n):
+ hasFuzz = true
}
}
- // Tests imply imported "testing" package in the code.
- // If there is no import, bail to let the compiler produce an error.
- if !testingImported && len(tests) > 0 {
- return nil
+ if hasTest || hasFuzz {
+ return true
}
- // We emulate "go test". An example with no "Output" comment is compiled,
- // but not executed. An example with no text after "Output:" is compiled,
- // executed, and expected to produce no output.
- var ex []*doc.Example
- // exNoOutput indicates whether an example with no output is found.
- // We need to compile the program containing such an example even if there are no
- // other tests or examples.
- exNoOutput := false
- for _, e := range doc.Examples(f) {
- if e.Output != "" || e.EmptyOutput {
- ex = append(ex, e)
- }
- if e.Output == "" && !e.EmptyOutput {
- exNoOutput = true
- }
- }
-
- if len(tests) == 0 && len(ex) == 0 && !exNoOutput {
- return nil
- }
-
- if !testingImported && (len(ex) > 0 || exNoOutput) {
- // In case of the program with examples and no "testing" package imported,
- // add import after "package main" without modifying line numbers.
- importDecl := []byte(`;import "testing";`)
- src = bytes.Join([][]byte{src[:importPos], importDecl, src[importPos:]}, nil)
- }
-
- data := struct {
- Tests []string
- Examples []*doc.Example
- }{
- tests,
- ex,
- }
- code := new(bytes.Buffer)
- if err := testTmpl.Execute(code, data); err != nil {
- panic(err)
- }
- src = append(src, code.Bytes()...)
- return src
+ return len(doc.Examples(f)) > 0
}
-var testTmpl = template.Must(template.New("main").Parse(`
-func main() {
- matchAll := func(t string, pat string) (bool, error) { return true, nil }
- tests := []testing.InternalTest{
-{{range .Tests}}
- {"{{.}}", {{.}}},
-{{end}}
- }
- examples := []testing.InternalExample{
-{{range .Examples}}
- {"Example{{.Name}}", Example{{.Name}}, {{printf "%q" .Output}}, {{.Unordered}}},
-{{end}}
- }
- testing.Main(matchAll, tests, nil, examples)
-}
-`))
-
var failedTestPattern = "--- FAIL"
// compileAndRun tries to build and run a user program.
@@ -341,7 +276,7 @@
return nil, err
}
if br.errorMessage != "" {
- return &response{Errors: br.errorMessage}, nil
+ return &response{Errors: removeBanner(br.errorMessage)}, nil
}
execRes, err := sandboxRun(ctx, br.exePath, br.testParam)
@@ -425,11 +360,10 @@
defer br.cleanup()
var buildPkgArg = "."
if files.Num() == 1 && len(files.Data(progName)) > 0 {
- buildPkgArg = progName
src := files.Data(progName)
- if code := getTestProg(src); code != nil {
+ if isTestProg(src) {
br.testParam = "-test.v"
- files.AddFile(progName, code)
+ files.MvFile(progName, progTestName)
}
}
@@ -474,7 +408,15 @@
return nil, fmt.Errorf("error copying GOCACHE: %v", err)
}
- cmd := exec.Command("/usr/local/go-faketime/bin/go", "build", "-o", br.exePath, "-tags=faketime")
+ var goArgs []string
+ if br.testParam != "" {
+ goArgs = append(goArgs, "test", "-c")
+ } else {
+ goArgs = append(goArgs, "build")
+ }
+ goArgs = append(goArgs, "-o", br.exePath, "-tags=faketime")
+
+ cmd := exec.Command("/usr/local/go-faketime/bin/go", goArgs...)
cmd.Dir = tmpDir
cmd.Env = []string{"GOOS=linux", "GOARCH=amd64", "GOROOT=/usr/local/go-faketime"}
cmd.Env = append(cmd.Env, "GOCACHE="+goCache)
@@ -668,6 +610,16 @@
}
}
+// removeBanner remove package name banner
+func removeBanner(output string) string {
+ if strings.HasPrefix(output, "#") {
+ if nl := strings.Index(output, "\n"); nl != -1 {
+ output = output[nl+1:]
+ }
+ }
+ return output
+}
+
const healthProg = `
package main
diff --git a/sandbox_test.go b/sandbox_test.go
index f090a48..de390bf 100644
--- a/sandbox_test.go
+++ b/sandbox_test.go
@@ -59,6 +59,12 @@
{"Benchmark", Benchmark1IsABenchmark, true},
{"Benchmark", BenchmarkisNotABenchmark, false},
+
+ {"Fuzz", Fuzz, true},
+ {"Fuzz", Fuzz1IsAFuzz, true},
+ {"Fuzz", FuzzÑIsAFuzz, true},
+
+ {"Fuzz", FuzzisNotAFuzz, false},
} {
name := nameOf(t, tc.f)
t.Run(name, func(t *testing.T) {
@@ -108,7 +114,7 @@
panic("This is not a valid test function.")
}
-// Test11IsATest is a valid test function.
+// Test1IsATest is a valid test function.
func Test1IsATest(t *testing.T) {
}
@@ -181,3 +187,25 @@
func BenchmarkisNotABenchmark(b *testing.B) {
panic("This is not a valid benchmark function.")
}
+
+// FuzzisNotAFuzz is not a fuzz test function, despite appearances.
+//
+// Please ignore any lint or vet warnings for this function.
+func FuzzisNotAFuzz(f *testing.F) {
+ panic("This is not a valid fuzzing function.")
+}
+
+// Fuzz1IsAFuzz is a valid fuzz function.
+func Fuzz1IsAFuzz(f *testing.F) {
+ f.Skip()
+}
+
+// Fuzz is a fuzz with a minimal name.
+func Fuzz(f *testing.F) {
+ f.Skip()
+}
+
+// FuzzÑIsAFuzz is a fuzz with an interesting Unicode name.
+func FuzzÑIsAFuzz(f *testing.F) {
+ f.Skip()
+}
diff --git a/tests.go b/tests.go
index 3c07789..46731a6 100644
--- a/tests.go
+++ b/tests.go
@@ -274,7 +274,7 @@
func ExampleNotExecuted() {
// Output: it should not run
}
-`, want: "", errors: "./prog.go:4:20: undefined: testing\n"},
+`, want: "", errors: "./prog_test.go:4:20: undefined: testing\n"},
{
name: "test_with_import_ignored",
@@ -469,7 +469,7 @@
{
name: "compile_modules_with_vet",
withVet: true,
- wantVetErrors: "go: finding module for package github.com/bradfitz/iter\ngo: found github.com/bradfitz/iter in github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8\n# play\n./prog.go:6:2: fmt.Printf format %v reads arg #1, but call has 0 args\n",
+ wantVetErrors: "./prog.go:6:2: fmt.Printf format %v reads arg #1, but call has 0 args\n",
prog: `
package main
import ("fmt"; "github.com/bradfitz/iter")
@@ -581,4 +581,56 @@
fmt.Println(net.ParseIP("1.2.3.4"))
}
`, want: "1.2.3.4\n"},
+ {
+ name: "fuzz_executed",
+ prog: `
+package main
+
+import "testing"
+
+func FuzzSanity(f *testing.F) {
+ f.Add("a")
+ f.Fuzz(func(t *testing.T, v string) {
+ })
+}
+`, want: `=== RUN FuzzSanity
+=== RUN FuzzSanity/seed#0
+--- PASS: FuzzSanity (0.00s)
+ --- PASS: FuzzSanity/seed#0 (0.00s)
+PASS`},
+ {
+ name: "test_main",
+ prog: `
+package main
+
+import (
+ "os"
+ "testing"
+)
+
+func TestMain(m *testing.M) {
+ os.Exit(m.Run())
+}`, want: `testing: warning: no tests to run
+PASS`,
+ },
+ {
+ name: "multiple_files_no_banner",
+ prog: `
+package main
+
+func main() {
+ print()
+}
+
+-- foo.go --
+package main
+
+import "fmt"
+
+func print() {
+ =fmt.Println("Hello, playground")
+}
+`, errors: `./foo.go:6:2: syntax error: unexpected =, expecting }
+`,
+ },
}
diff --git a/txtar.go b/txtar.go
index 591dcbf..11f6ab4 100644
--- a/txtar.go
+++ b/txtar.go
@@ -48,6 +48,30 @@
}
}
+func (fs *fileSet) Update(filename string, src []byte) {
+ if fs.Contains(filename) {
+ fs.m[filename] = src
+ }
+}
+
+func (fs *fileSet) MvFile(source, target string) {
+ if fs.m == nil {
+ return
+ }
+ data, ok := fs.m[source]
+ if !ok {
+ return
+ }
+ fs.m[target] = data
+ delete(fs.m, source)
+ for i := range fs.files {
+ if fs.files[i] == source {
+ fs.files[i] = target
+ break
+ }
+ }
+}
+
// Format returns fs formatted as a txtar archive.
func (fs *fileSet) Format() []byte {
a := new(txtar.Archive)
diff --git a/vet.go b/vet.go
index ece62a7..d58c0f7 100644
--- a/vet.go
+++ b/vet.go
@@ -83,12 +83,6 @@
// Rewrite compiler errors to refer to progName
// instead of '/tmp/sandbox1234/main.go'.
errs := strings.Replace(string(out), dir, "", -1)
-
- // Remove vet's package name banner.
- if strings.HasPrefix(errs, "#") {
- if nl := strings.Index(errs, "\n"); nl != -1 {
- errs = errs[nl+1:]
- }
- }
+ errs = removeBanner(errs)
return errs, nil
}
