go/analysis/unitchecker: Example of separate analysis
This change adds an Example that demonstrates the
principle of separate analysis by having one program
(the manager) visit the import graph in postorder,
invoking a second program (the worker) as a child
process once per package. The manager is analogous
to go vet, and the worker to the vettool.
Because unitchecker is closely coupled to go vet,
both in its config file, and in assumptions about
where and how facts and types are produced, this
change parameterizes unitchecker to illustrate
that not just facts but types too may be produced
by the worker, without the need for the compiler.
This may be a simpler and more efficient design
for (say) a distributed analysis system.
There are no changes to the public API yet
(a proposal will follow) so the example can't
yet serve directly as the basis for a new
implementation, but in the meantime it should
at least illuminate what is involved and,
perhaps, how unitchecker can be easily forked
and adapted.
The changes to unitchecker should not affect the
behavior of existing programs.
Change-Id: I4ae414ccde91853c77b8a45771dd6e5cfc015173
Reviewed-on: https://go-review.googlesource.com/c/tools/+/510215
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Robert Findley <rfindley@google.com>
Auto-Submit: Alan Donovan <adonovan@google.com>
Run-TryBot: Alan Donovan <adonovan@google.com>
diff --git a/go/analysis/unitchecker/separate_test.go b/go/analysis/unitchecker/separate_test.go
new file mode 100644
index 0000000..12d9104
--- /dev/null
+++ b/go/analysis/unitchecker/separate_test.go
@@ -0,0 +1,302 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package unitchecker_test
+
+// This file illustrates separate analysis with an example.
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "go/token"
+ "go/types"
+ "io"
+ "log"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "sync/atomic"
+
+ "golang.org/x/tools/go/analysis/passes/printf"
+ "golang.org/x/tools/go/analysis/unitchecker"
+ "golang.org/x/tools/go/gcexportdata"
+ "golang.org/x/tools/go/packages"
+ "golang.org/x/tools/txtar"
+)
+
+// ExampleSeparateAnalysis demonstrates the principle of separate
+// analysis, the distribution of units of type-checking and analysis
+// work across several processes, using serialized summaries to
+// communicate between them.
+//
+// It uses two different kinds of task, "manager" and "worker":
+//
+// - The manager computes the graph of package dependencies, and makes
+// a request to the worker for each package. It does not parse,
+// type-check, or analyze Go code. It is analogous "go vet".
+//
+// - The worker, which contains the Analyzers, reads each request,
+// loads, parses, and type-checks the files of one package,
+// applies all necessary analyzers to the package, then writes
+// its results to a file. It is a unitchecker-based driver,
+// analogous to the program specified by go vet -vettool= flag.
+//
+// In practice these would be separate executables, but for simplicity
+// of this example they are provided by one executable in two
+// different modes: the Example function is the manager, and the same
+// executable invoked with ENTRYPOINT=worker is the worker.
+// (See TestIntegration for how this happens.)
+func ExampleSeparateAnalysis() {
+ // src is an archive containing a module with a printf mistake.
+ const src = `
+-- go.mod --
+module separate
+go 1.18
+
+-- main/main.go --
+package main
+
+import "separate/lib"
+
+func main() {
+ lib.MyPrintf("%s", 123)
+}
+
+-- lib/lib.go --
+package lib
+
+import "fmt"
+
+func MyPrintf(format string, args ...any) {
+ fmt.Printf(format, args...)
+}
+`
+
+ // Expand archive into tmp tree.
+ tmpdir, err := os.MkdirTemp("", "SeparateAnalysis")
+ if err != nil {
+ log.Fatal(err)
+ }
+ if err := extractTxtar(txtar.Parse([]byte(src)), tmpdir); err != nil {
+ log.Fatal(err)
+ }
+
+ // Load metadata for the main package and all its dependencies.
+ cfg := &packages.Config{
+ Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | packages.NeedImports | packages.NeedModule,
+ Dir: tmpdir,
+ Env: append(os.Environ(),
+ "GOPROXY=off", // disable network
+ "GOWORK=off", // an ambient GOWORK value would break package loading
+ ),
+ }
+ pkgs, err := packages.Load(cfg, "separate/main")
+ if err != nil {
+ log.Fatal(err)
+ }
+ // Stop if any package had a metadata error.
+ if packages.PrintErrors(pkgs) > 0 {
+ os.Exit(1)
+ }
+
+ // Now we have loaded the import graph,
+ // let's begin the proper work of the manager.
+
+ // Gather root packages. They will get all analyzers,
+ // whereas dependencies get only the subset that
+ // produce facts or are required by them.
+ roots := make(map[*packages.Package]bool)
+ for _, pkg := range pkgs {
+ roots[pkg] = true
+ }
+
+ // nextID generates sequence numbers for each unit of work.
+ // We use it to create names of temporary files.
+ var nextID atomic.Int32
+
+ // Visit all packages in postorder: dependencies first.
+ // TODO(adonovan): opt: use parallel postorder.
+ packages.Visit(pkgs, nil, func(pkg *packages.Package) {
+ if pkg.PkgPath == "unsafe" {
+ return
+ }
+
+ // Choose a unique prefix for temporary files
+ // (.cfg .types .facts) produced by this package.
+ // We stow it in an otherwise unused field of
+ // Package so it can be accessed by our importers.
+ prefix := fmt.Sprintf("%s/%d", tmpdir, nextID.Add(1))
+ pkg.ExportFile = prefix
+
+ // Construct the request to the worker.
+ var (
+ importMap = make(map[string]string)
+ packageFile = make(map[string]string)
+ packageVetx = make(map[string]string)
+ )
+ for importPath, dep := range pkg.Imports {
+ importMap[importPath] = dep.PkgPath
+ if depPrefix := dep.ExportFile; depPrefix != "" { // skip "unsafe"
+ packageFile[dep.PkgPath] = depPrefix + ".types"
+ packageVetx[dep.PkgPath] = depPrefix + ".facts"
+ }
+ }
+ cfg := unitchecker.Config{
+ ID: pkg.ID,
+ ImportPath: pkg.PkgPath,
+ GoFiles: pkg.CompiledGoFiles,
+ NonGoFiles: pkg.OtherFiles,
+ IgnoredFiles: pkg.IgnoredFiles,
+ ImportMap: importMap,
+ PackageFile: packageFile,
+ PackageVetx: packageVetx,
+ VetxOnly: !roots[pkg],
+ VetxOutput: prefix + ".facts",
+ }
+ if pkg.Module != nil {
+ if v := pkg.Module.GoVersion; v != "" {
+ cfg.GoVersion = "go" + v
+ }
+ }
+
+ // Write the JSON configuration message to a file.
+ cfgData, err := json.Marshal(cfg)
+ if err != nil {
+ log.Fatal(err)
+ }
+ cfgFile := prefix + ".cfg"
+ if err := os.WriteFile(cfgFile, cfgData, 0666); err != nil {
+ log.Fatal(err)
+ }
+
+ // Send the request to the worker.
+ cmd := exec.Command(os.Args[0], "-json", cfgFile)
+ cmd.Stderr = os.Stderr
+ cmd.Stdout = new(bytes.Buffer)
+ cmd.Env = append(os.Environ(), "ENTRYPOINT=worker")
+ if err := cmd.Run(); err != nil {
+ log.Fatal(err)
+ }
+
+ // Parse JSON output and print plainly.
+ dec := json.NewDecoder(cmd.Stdout.(io.Reader))
+ for {
+ type jsonDiagnostic struct {
+ Posn string `json:"posn"`
+ Message string `json:"message"`
+ }
+ // 'results' maps Package.Path -> Analyzer.Name -> diagnostics
+ var results map[string]map[string][]jsonDiagnostic
+ if err := dec.Decode(&results); err != nil {
+ if err == io.EOF {
+ break
+ }
+ log.Fatal(err)
+ }
+ for _, result := range results {
+ for analyzer, diags := range result {
+ for _, diag := range diags {
+ rel := strings.ReplaceAll(diag.Posn, tmpdir, "")
+ rel = filepath.ToSlash(rel)
+ fmt.Printf("%s: [%s] %s\n",
+ rel, analyzer, diag.Message)
+ }
+ }
+ }
+ }
+ })
+
+ // Observe that the example produces a fact-based diagnostic
+ // from separate analysis of "main", "lib", and "fmt":
+
+ // Output:
+ // /main/main.go:6:2: [printf] separate/lib.MyPrintf format %s has arg 123 of wrong type int
+}
+
+// -- worker process --
+
+// worker is the main entry point for a unitchecker-based driver
+// with only a single analyzer, for illustration.
+func worker() {
+ // Currently the unitchecker API doesn't allow clients to
+ // control exactly how and where fact and type information
+ // is produced and consumed.
+ //
+ // So, for example, it assumes that type information has
+ // already been produced by the compiler, which is true when
+ // running under "go vet", but isn't necessary. It may be more
+ // convenient and efficient for a distributed analysis system
+ // if the worker generates both of them, which is the approach
+ // taken in this example; they could even be saved as two
+ // sections of a single file.
+ //
+ // Consequently, this test currently needs special access to
+ // private hooks in unitchecker to control how and where facts
+ // and types are produced and consumed. In due course this
+ // will become a respectable public API. In the meantime, it
+ // should at least serve as a demonstration of how one could
+ // fork unitchecker to achieve separate analysis without go vet.
+ unitchecker.SetTypeImportExport(makeTypesImporter, exportTypes)
+
+ unitchecker.Main(printf.Analyzer)
+}
+
+func makeTypesImporter(cfg *unitchecker.Config, fset *token.FileSet) types.Importer {
+ imports := make(map[string]*types.Package)
+ return importerFunc(func(importPath string) (*types.Package, error) {
+ // Resolve import path to package path (vendoring, etc)
+ path, ok := cfg.ImportMap[importPath]
+ if !ok {
+ return nil, fmt.Errorf("can't resolve import %q", path)
+ }
+ if path == "unsafe" {
+ return types.Unsafe, nil
+ }
+
+ // Find, read, and decode file containing type information.
+ file, ok := cfg.PackageFile[path]
+ if !ok {
+ return nil, fmt.Errorf("no package file for %q", path)
+ }
+ f, err := os.Open(file)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close() // ignore error
+ return gcexportdata.Read(f, fset, imports, path)
+ })
+}
+
+func exportTypes(cfg *unitchecker.Config, fset *token.FileSet, pkg *types.Package) error {
+ var out bytes.Buffer
+ if err := gcexportdata.Write(&out, fset, pkg); err != nil {
+ return err
+ }
+ typesFile := strings.TrimSuffix(cfg.VetxOutput, ".facts") + ".types"
+ return os.WriteFile(typesFile, out.Bytes(), 0666)
+}
+
+// -- helpers --
+
+type importerFunc func(path string) (*types.Package, error)
+
+func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) }
+
+// extractTxtar writes each archive file to the corresponding location beneath dir.
+//
+// TODO(adonovan): move this to txtar package, we need it all the time (#61386).
+func extractTxtar(ar *txtar.Archive, dir string) error {
+ for _, file := range ar.Files {
+ name := filepath.Join(dir, file.Name)
+ if err := os.MkdirAll(filepath.Dir(name), 0777); err != nil {
+ return err
+ }
+ if err := os.WriteFile(name, file.Data, 0666); err != nil {
+ return err
+ }
+ }
+ return nil
+}
