blob: 4e5b413f3ff149a8b2c6d15a5fcfd6e9bfdfafe9 [file] [log] [blame]
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Program for performing test runs using "fuzz-driver".
// Main loop iteratively runs "fuzz-driver" to create a corpus,
// then builds and runs the code. If a failure in the run is
// detected, then a testcase minimization phase kicks in.
package main
import (
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"strings"
"time"
generator "golang.org/x/tools/cmd/signature-fuzzer/internal/fuzz-generator"
)
const pkName = "fzTest"
// Basic options
var verbflag = flag.Int("v", 0, "Verbose trace output level")
var loopitflag = flag.Int("numit", 10, "Number of main loop iterations to run")
var seedflag = flag.Int64("seed", -1, "Random seed")
var execflag = flag.Bool("execdriver", false, "Exec fuzz-driver binary instead of invoking generator directly")
var numpkgsflag = flag.Int("numpkgs", 50, "Number of test packages")
var numfcnsflag = flag.Int("numfcns", 20, "Number of test functions per package.")
// Debugging/testing options. These tell the generator to emit "bad" code so as to
// test the logic for detecting errors and/or minimization.
var emitbadflag = flag.Int("emitbad", -1, "[Testing only] force generator to emit 'bad' code.")
var selbadpkgflag = flag.Int("badpkgidx", 0, "[Testing only] select index of bad package (used with -emitbad)")
var selbadfcnflag = flag.Int("badfcnidx", 0, "[Testing only] select index of bad function (used with -emitbad)")
var forcetmpcleanflag = flag.Bool("forcetmpclean", false, "[Testing only] force cleanup of temp dir")
var cleancacheflag = flag.Bool("cleancache", true, "[Testing only] don't clean the go cache")
var raceflag = flag.Bool("race", false, "[Testing only] build generated code with -race")
func verb(vlevel int, s string, a ...interface{}) {
if *verbflag >= vlevel {
fmt.Printf(s, a...)
fmt.Printf("\n")
}
}
func warn(s string, a ...interface{}) {
fmt.Fprintf(os.Stderr, s, a...)
fmt.Fprintf(os.Stderr, "\n")
}
func fatal(s string, a ...interface{}) {
fmt.Fprintf(os.Stderr, s, a...)
fmt.Fprintf(os.Stderr, "\n")
os.Exit(1)
}
type config struct {
generator.GenConfig
tmpdir string
gendir string
buildOutFile string
runOutFile string
gcflags string
nerrors int
}
func usage(msg string) {
if len(msg) > 0 {
fmt.Fprintf(os.Stderr, "error: %s\n", msg)
}
fmt.Fprintf(os.Stderr, "usage: fuzz-runner [flags]\n\n")
flag.PrintDefaults()
fmt.Fprintf(os.Stderr, "Example:\n\n")
fmt.Fprintf(os.Stderr, " fuzz-runner -numit=500 -numpkgs=11 -numfcns=13 -seed=10101\n\n")
fmt.Fprintf(os.Stderr, " \tRuns 500 rounds of test case generation\n")
fmt.Fprintf(os.Stderr, " \tusing random see 10101, in each round emitting\n")
fmt.Fprintf(os.Stderr, " \t11 packages each with 13 function pairs.\n")
os.Exit(2)
}
// docmd executes the specified command in the dir given and pipes the
// output to stderr. return status is 0 if command passed, 1
// otherwise.
func docmd(cmd []string, dir string) int {
verb(2, "docmd: %s", strings.Join(cmd, " "))
c := exec.Command(cmd[0], cmd[1:]...)
if dir != "" {
c.Dir = dir
}
b, err := c.CombinedOutput()
st := 0
if err != nil {
warn("error executing cmd %s: %v",
strings.Join(cmd, " "), err)
st = 1
}
os.Stderr.Write(b)
return st
}
// docodmout forks and execs command 'cmd' in dir 'dir', redirecting
// stderr and stdout from the execution to file 'outfile'.
func docmdout(cmd []string, dir string, outfile string) int {
of, err := os.OpenFile(outfile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
fatal("opening outputfile %s: %v", outfile, err)
}
c := exec.Command(cmd[0], cmd[1:]...)
defer of.Close()
if dir != "" {
verb(2, "setting cmd.Dir to %s", dir)
c.Dir = dir
}
verb(2, "docmdout: %s > %s", strings.Join(cmd, " "), outfile)
c.Stdout = of
c.Stderr = of
err = c.Run()
st := 0
if err != nil {
warn("error executing cmd %s: %v",
strings.Join(cmd, " "), err)
st = 1
}
return st
}
// gen is the main hook for kicking off code generation. For
// non-minimization runs, 'singlepk' and 'singlefn' will both be -1
// (indicating that we want all functions and packages to be
// generated). If 'singlepk' is set to a non-negative value, then
// code generation will be restricted to the single package with that
// index (as a try at minimization), similarly with 'singlefn'
// restricting the codegen to a single specified function.
func (c *config) gen(singlepk int, singlefn int) {
// clean the output dir
verb(2, "cleaning outdir %s", c.gendir)
if err := os.RemoveAll(c.gendir); err != nil {
fatal("error cleaning gen dir %s: %v", c.gendir, err)
}
// emit code into the output dir. Here we either invoke the
// generator directly, or invoke fuzz-driver if -execflag is
// set. If the code generation process itself fails, this is
// typically a bug in the fuzzer itself, so it gets reported
// as a fatal error.
if *execflag {
args := []string{"fuzz-driver",
"-numpkgs", strconv.Itoa(c.NumTestPackages),
"-numfcns", strconv.Itoa(c.NumTestFunctions),
"-seed", strconv.Itoa(int(c.Seed)),
"-outdir", c.OutDir,
"-pkgpath", pkName,
"-maxfail", strconv.Itoa(c.MaxFail)}
if singlepk != -1 {
args = append(args, "-pkgmask", strconv.Itoa(singlepk))
}
if singlefn != -1 {
args = append(args, "-fcnmask", strconv.Itoa(singlefn))
}
if *emitbadflag != 0 {
args = append(args, "-emitbad", strconv.Itoa(*emitbadflag),
"-badpkgidx", strconv.Itoa(*selbadpkgflag),
"-badfcnidx", strconv.Itoa(*selbadfcnflag))
}
verb(1, "invoking fuzz-driver with args: %v", args)
st := docmd(args, "")
if st != 0 {
fatal("fatal error: generation failed, cmd was: %v", args)
}
} else {
if singlepk != -1 {
c.PkgMask = map[int]int{singlepk: 1}
}
if singlefn != -1 {
c.FcnMask = map[int]int{singlefn: 1}
}
verb(1, "invoking generator.Generate with config: %v", c.GenConfig)
errs := generator.Generate(c.GenConfig)
if errs != 0 {
log.Fatal("errors during generation")
}
}
}
// action performs a selected action/command in the generated code dir.
func (c *config) action(cmd []string, outfile string, emitout bool) int {
st := docmdout(cmd, c.gendir, outfile)
if emitout {
content, err := ioutil.ReadFile(outfile)
if err != nil {
log.Fatal(err)
}
fmt.Fprintf(os.Stderr, "%s", content)
}
return st
}
func binaryName() string {
if runtime.GOOS == "windows" {
return pkName + ".exe"
} else {
return "./" + pkName
}
}
// build builds a generated corpus of Go code. If 'emitout' is set, then dump out the
// results of the build after it completes (during minimization emitout is set to false,
// since there is no need to see repeated errors).
func (c *config) build(emitout bool) int {
// Issue a build of the generated code.
c.buildOutFile = filepath.Join(c.tmpdir, "build.err.txt")
cmd := []string{"go", "build", "-o", binaryName()}
if c.gcflags != "" {
cmd = append(cmd, "-gcflags=all="+c.gcflags)
}
if *raceflag {
cmd = append(cmd, "-race")
}
cmd = append(cmd, ".")
verb(1, "build command is: %v", cmd)
return c.action(cmd, c.buildOutFile, emitout)
}
// run invokes a binary built from a generated corpus of Go code. If
// 'emitout' is set, then dump out the results of the run after it
// completes.
func (c *config) run(emitout bool) int {
// Issue a run of the generated code.
c.runOutFile = filepath.Join(c.tmpdir, "run.err.txt")
cmd := []string{filepath.Join(c.gendir, binaryName())}
verb(1, "run command is: %v", cmd)
return c.action(cmd, c.runOutFile, emitout)
}
type minimizeMode int
const (
minimizeBuildFailure = iota
minimizeRuntimeFailure
)
// minimize tries to minimize a failing scenario down to a single
// package and/or function if possible. This is done using an
// iterative search. Here 'minimizeMode' tells us whether we're
// looking for a compile-time error or a runtime error.
func (c *config) minimize(mode minimizeMode) int {
verb(0, "... starting minimization for failed directory %s", c.gendir)
foundPkg := -1
foundFcn := -1
// Locate bad package. Uses brute-force linear search, could do better...
for pidx := 0; pidx < c.NumTestPackages; pidx++ {
verb(1, "minimization: trying package %d", pidx)
c.gen(pidx, -1)
st := c.build(false)
if mode == minimizeBuildFailure {
if st != 0 {
// Found.
foundPkg = pidx
c.nerrors++
break
}
} else {
if st != 0 {
warn("run minimization: unexpected build failed while searching for bad pkg")
return 1
}
st := c.run(false)
if st != 0 {
// Found.
c.nerrors++
verb(1, "run minimization found bad package: %d", pidx)
foundPkg = pidx
break
}
}
}
if foundPkg == -1 {
verb(0, "** minimization failed, could not locate bad package")
return 1
}
warn("package minimization succeeded: found bad pkg %d", foundPkg)
// clean unused packages
for pidx := 0; pidx < c.NumTestPackages; pidx++ {
if pidx != foundPkg {
chp := filepath.Join(c.gendir, fmt.Sprintf("%s%s%d", c.Tag, generator.CheckerName, pidx))
if err := os.RemoveAll(chp); err != nil {
fatal("failed to clean pkg subdir %s: %v", chp, err)
}
clp := filepath.Join(c.gendir, fmt.Sprintf("%s%s%d", c.Tag, generator.CallerName, pidx))
if err := os.RemoveAll(clp); err != nil {
fatal("failed to clean pkg subdir %s: %v", clp, err)
}
}
}
// Locate bad function. Again, brute force.
for fidx := 0; fidx < c.NumTestFunctions; fidx++ {
c.gen(foundPkg, fidx)
st := c.build(false)
if mode == minimizeBuildFailure {
if st != 0 {
// Found.
verb(1, "build minimization found bad function: %d", fidx)
foundFcn = fidx
break
}
} else {
if st != 0 {
warn("run minimization: unexpected build failed while searching for bad fcn")
return 1
}
st := c.run(false)
if st != 0 {
// Found.
verb(1, "run minimization found bad function: %d", fidx)
foundFcn = fidx
break
}
}
// not the function we want ... continue the hunt
}
if foundFcn == -1 {
verb(0, "** function minimization failed, could not locate bad function")
return 1
}
warn("function minimization succeeded: found bad fcn %d", foundFcn)
return 0
}
// cleanTemp removes the temp dir we've been working with.
func (c *config) cleanTemp() {
if !*forcetmpcleanflag {
if c.nerrors != 0 {
verb(1, "preserving temp dir %s", c.tmpdir)
return
}
}
verb(1, "cleaning temp dir %s", c.tmpdir)
os.RemoveAll(c.tmpdir)
}
// perform is the top level driver routine for the program, containing the
// main loop. Each iteration of the loop performs a generate/build/run
// sequence, and then updates the seed afterwards if no failure is found.
// If a failure is detected, we try to minimize it and then return without
// attempting any additional tests.
func (c *config) perform() int {
defer c.cleanTemp()
// Main loop
for iter := 0; iter < *loopitflag; iter++ {
if iter != 0 && iter%50 == 0 {
// Note: cleaning the Go cache periodically is
// pretty much a requirement if you want to do
// things like overnight runs of the fuzzer,
// but it is also a very unfriendly thing do
// to if we're executing as part of a unit
// test run (in which case there may be other
// tests running in parallel with this
// one). Check the "cleancache" flag before
// doing this.
if *cleancacheflag {
docmd([]string{"go", "clean", "-cache"}, "")
}
}
verb(0, "... begin iteration %d with current seed %d", iter, c.Seed)
c.gen(-1, -1)
st := c.build(true)
if st != 0 {
c.minimize(minimizeBuildFailure)
return 1
}
st = c.run(true)
if st != 0 {
c.minimize(minimizeRuntimeFailure)
return 1
}
// update seed so that we get different code on the next iter.
c.Seed += 101
}
return 0
}
func main() {
log.SetFlags(0)
log.SetPrefix("fuzz-runner: ")
flag.Parse()
if flag.NArg() != 0 {
usage("unknown extra arguments")
}
verb(1, "in main, verblevel=%d", *verbflag)
tmpdir, err := ioutil.TempDir("", "fuzzrun")
if err != nil {
fatal("creation of tempdir failed: %v", err)
}
gendir := filepath.Join(tmpdir, "fuzzTest")
// select starting seed
if *seedflag == -1 {
now := time.Now()
*seedflag = now.UnixNano() % 123456789
}
// set up params for this run
c := &config{
GenConfig: generator.GenConfig{
NumTestPackages: *numpkgsflag, // 100
NumTestFunctions: *numfcnsflag, // 20
Seed: *seedflag,
OutDir: gendir,
Pragma: "-maxfail=9999",
PkgPath: pkName,
EmitBad: *emitbadflag,
BadPackageIdx: *selbadpkgflag,
BadFuncIdx: *selbadfcnflag,
},
tmpdir: tmpdir,
gendir: gendir,
}
// kick off the main loop.
st := c.perform()
// done
verb(1, "leaving main, num errors=%d", c.nerrors)
os.Exit(st)
}