go/analysis/unitchecker/separate_test.go - tools - Git at Google

 // Copyright 2023 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 package unitchecker_test

 // This file illustrates separate analysis with an example.

 import (
 	"bytes"
 	"encoding/json"
 	"fmt"
 	"go/token"
 	"go/types"
 	"io"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 	"sync/atomic"
 	"testing"

 	"golang.org/x/tools/go/analysis/passes/printf"
 	"golang.org/x/tools/go/analysis/unitchecker"
 	"golang.org/x/tools/go/gcexportdata"
 	"golang.org/x/tools/go/packages"
 	"golang.org/x/tools/internal/testenv"
 	"golang.org/x/tools/internal/testfiles"
 	"golang.org/x/tools/txtar"
 )

 // TestExampleSeparateAnalysis demonstrates the principle of separate
 // analysis, the distribution of units of type-checking and analysis
 // work across several processes, using serialized summaries to
 // communicate between them.
 //
 // It uses two different kinds of task, "manager" and "worker":
 //
 //   - The manager computes the graph of package dependencies, and makes
 //     a request to the worker for each package. It does not parse,
 //     type-check, or analyze Go code. It is analogous "go vet".
 //
 //   - The worker, which contains the Analyzers, reads each request,
 //     loads, parses, and type-checks the files of one package,
 //     applies all necessary analyzers to the package, then writes
 //     its results to a file. It is a unitchecker-based driver,
 //     analogous to the program specified by go vet -vettool= flag.
 //
 // In practice these would be separate executables, but for simplicity
 // of this example they are provided by one executable in two
 // different modes: the Example function is the manager, and the same
 // executable invoked with ENTRYPOINT=worker is the worker.
 // (See TestIntegration for how this happens.)
 //
 // Unfortunately this can't be a true Example because of the skip,
 // which requires a testing.T.
 func TestExampleSeparateAnalysis(t *testing.T) {
 	testenv.NeedsGoPackages(t)

 	// src is an archive containing a module with a printf mistake.
 	const src = `
 -- go.mod --
 module separate
 go 1.18

 -- main/main.go --
 package main

 import "separate/lib"

 func main() {
 	lib.MyPrintf("%s", 123)
 }

 -- lib/lib.go --
 package lib

 import "fmt"

 func MyPrintf(format string, args ...any) {
 	fmt.Printf(format, args...)
 }
 `

 	// Expand archive into tmp tree.
 	fs, err := txtar.FS(txtar.Parse([]byte(src)))
 	if err != nil {
 		t.Fatal(err)
 	}
 	tmpdir := testfiles.CopyToTmp(t, fs)

 	// Load metadata for the main package and all its dependencies.
 	cfg := &packages.Config{
 		Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | packages.NeedImports | packages.NeedModule,
 		Dir:  tmpdir,
 		Env: append(os.Environ(),
 			"GOPROXY=off", // disable network
 			"GOWORK=off",  // an ambient GOWORK value would break package loading
 		),
 		Logf: t.Logf,
 	}
 	pkgs, err := packages.Load(cfg, "separate/main")
 	if err != nil {
 		t.Fatal(err)
 	}
 	// Stop if any package had a metadata error.
 	if packages.PrintErrors(pkgs) > 0 {
 		t.Fatal("there were errors among loaded packages")
 	}

 	// Now we have loaded the import graph,
 	// let's begin the proper work of the manager.

 	// Gather root packages. They will get all analyzers,
 	// whereas dependencies get only the subset that
 	// produce facts or are required by them.
 	roots := make(map[*packages.Package]bool)
 	for _, pkg := range pkgs {
 		roots[pkg] = true
 	}

 	// nextID generates sequence numbers for each unit of work.
 	// We use it to create names of temporary files.
 	var nextID atomic.Int32

 	var allDiagnostics []string

 	// Visit all packages in postorder: dependencies first.
 	// TODO(adonovan): opt: use parallel postorder.
 	packages.Visit(pkgs, nil, func(pkg *packages.Package) {
 		if pkg.PkgPath == "unsafe" {
 			return
 		}

 		// Choose a unique prefix for temporary files
 		// (.cfg .types .facts) produced by this package.
 		// We stow it in an otherwise unused field of
 		// Package so it can be accessed by our importers.
 		prefix := fmt.Sprintf("%s/%d", tmpdir, nextID.Add(1))
 		pkg.ExportFile = prefix

 		// Construct the request to the worker.
 		var (
 			importMap   = make(map[string]string)
 			packageFile = make(map[string]string)
 			packageVetx = make(map[string]string)
 		)
 		for importPath, dep := range pkg.Imports {
 			importMap[importPath] = dep.PkgPath
 			if depPrefix := dep.ExportFile; depPrefix != "" { // skip "unsafe"
 				packageFile[dep.PkgPath] = depPrefix + ".types"
 				packageVetx[dep.PkgPath] = depPrefix + ".facts"
 			}
 		}
 		cfg := unitchecker.Config{
 			ID:           pkg.ID,
 			ImportPath:   pkg.PkgPath,
 			GoFiles:      pkg.CompiledGoFiles,
 			NonGoFiles:   pkg.OtherFiles,
 			IgnoredFiles: pkg.IgnoredFiles,
 			ImportMap:    importMap,
 			PackageFile:  packageFile,
 			PackageVetx:  packageVetx,
 			VetxOnly:     !roots[pkg],
 			VetxOutput:   prefix + ".facts",
 		}
 		if pkg.Module != nil {
 			if v := pkg.Module.GoVersion; v != "" {
 				cfg.GoVersion = "go" + v
 			}
 			cfg.ModulePath = pkg.Module.Path
 			cfg.ModuleVersion = pkg.Module.Version
 		}

 		// Write the JSON configuration message to a file.
 		cfgData, err := json.Marshal(cfg)
 		if err != nil {
 			t.Fatalf("internal error in json.Marshal: %v", err)
 		}
 		cfgFile := prefix + ".cfg"
 		if err := os.WriteFile(cfgFile, cfgData, 0666); err != nil {
 			t.Fatal(err)
 		}

 		// Send the request to the worker.
 		cmd := testenv.Command(t, os.Args[0], "-json", cfgFile)
 		cmd.Stderr = os.Stderr
 		cmd.Stdout = new(bytes.Buffer)
 		cmd.Env = append(os.Environ(), "ENTRYPOINT=worker")
 		if err := cmd.Run(); err != nil {
 			t.Fatal(err)
 		}

 		// Parse JSON output and gather in allDiagnostics.
 		dec := json.NewDecoder(cmd.Stdout.(io.Reader))
 		for {
 			type jsonDiagnostic struct {
 				Posn    string `json:"posn"`
 				Message string `json:"message"`
 			}
 			// 'results' maps Package.Path -> Analyzer.Name -> diagnostics
 			var results map[string]map[string][]jsonDiagnostic
 			if err := dec.Decode(&results); err != nil {
 				if err == io.EOF {
 					break
 				}
 				t.Fatalf("internal error decoding JSON: %v", err)
 			}
 			for _, result := range results {
 				for analyzer, diags := range result {
 					for _, diag := range diags {
 						rel := strings.ReplaceAll(diag.Posn, tmpdir, "")
 						rel = filepath.ToSlash(rel)
 						msg := fmt.Sprintf("%s: [%s] %s", rel, analyzer, diag.Message)
 						allDiagnostics = append(allDiagnostics, msg)
 					}
 				}
 			}
 		}
 	})

 	// Observe that the example produces a fact-based diagnostic
 	// from separate analysis of "main", "lib", and "fmt":

 	const want = `/main/main.go:6:2: [printf] separate/lib.MyPrintf format %s has arg 123 of wrong type int`
 	sort.Strings(allDiagnostics)
 	if got := strings.Join(allDiagnostics, "\n"); got != want {
 		t.Errorf("Got: %s\nWant: %s", got, want)
 	}
 }

 // -- worker process --

 // worker is the main entry point for a unitchecker-based driver
 // with only a single analyzer, for illustration.
 func worker() {
 	// Currently the unitchecker API doesn't allow clients to
 	// control exactly how and where fact and type information
 	// is produced and consumed.
 	//
 	// So, for example, it assumes that type information has
 	// already been produced by the compiler, which is true when
 	// running under "go vet", but isn't necessary. It may be more
 	// convenient and efficient for a distributed analysis system
 	// if the worker generates both of them, which is the approach
 	// taken in this example; they could even be saved as two
 	// sections of a single file.
 	//
 	// Consequently, this test currently needs special access to
 	// private hooks in unitchecker to control how and where facts
 	// and types are produced and consumed. In due course this
 	// will become a respectable public API. In the meantime, it
 	// should at least serve as a demonstration of how one could
 	// fork unitchecker to achieve separate analysis without go vet.
 	unitchecker.SetTypeImportExport(makeTypesImporter, exportTypes)

 	unitchecker.Main(printf.Analyzer)
 }

 func makeTypesImporter(cfg *unitchecker.Config, fset *token.FileSet) types.Importer {
 	imports := make(map[string]*types.Package)
 	return importerFunc(func(importPath string) (*types.Package, error) {
 		// Resolve import path to package path (vendoring, etc)
 		path, ok := cfg.ImportMap[importPath]
 		if !ok {
 			return nil, fmt.Errorf("can't resolve import %q", path)
 		}
 		if path == "unsafe" {
 			return types.Unsafe, nil
 		}

 		// Find, read, and decode file containing type information.
 		file, ok := cfg.PackageFile[path]
 		if !ok {
 			return nil, fmt.Errorf("no package file for %q", path)
 		}
 		f, err := os.Open(file)
 		if err != nil {
 			return nil, err
 		}
 		defer f.Close() // ignore error
 		return gcexportdata.Read(f, fset, imports, path)
 	})
 }

 func exportTypes(cfg *unitchecker.Config, fset *token.FileSet, pkg *types.Package) error {
 	var out bytes.Buffer
 	if err := gcexportdata.Write(&out, fset, pkg); err != nil {
 		return err
 	}
 	typesFile := strings.TrimSuffix(cfg.VetxOutput, ".facts") + ".types"
 	return os.WriteFile(typesFile, out.Bytes(), 0666)
 }

 // -- helpers --

 type importerFunc func(path string) (*types.Package, error)

 func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) }
	// Copyright 2023 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	package unitchecker_test

	// This file illustrates separate analysis with an example.

	import (
	"bytes"
	"encoding/json"
	"fmt"
	"go/token"
	"go/types"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync/atomic"
	"testing"

	"golang.org/x/tools/go/analysis/passes/printf"
	"golang.org/x/tools/go/analysis/unitchecker"
	"golang.org/x/tools/go/gcexportdata"
	"golang.org/x/tools/go/packages"
	"golang.org/x/tools/internal/testenv"
	"golang.org/x/tools/internal/testfiles"
	"golang.org/x/tools/txtar"
	)

	// TestExampleSeparateAnalysis demonstrates the principle of separate
	// analysis, the distribution of units of type-checking and analysis
	// work across several processes, using serialized summaries to
	// communicate between them.
	//
	// It uses two different kinds of task, "manager" and "worker":
	//
	// - The manager computes the graph of package dependencies, and makes
	// a request to the worker for each package. It does not parse,
	// type-check, or analyze Go code. It is analogous "go vet".
	//
	// - The worker, which contains the Analyzers, reads each request,
	// loads, parses, and type-checks the files of one package,
	// applies all necessary analyzers to the package, then writes
	// its results to a file. It is a unitchecker-based driver,
	// analogous to the program specified by go vet -vettool= flag.
	//
	// In practice these would be separate executables, but for simplicity
	// of this example they are provided by one executable in two
	// different modes: the Example function is the manager, and the same
	// executable invoked with ENTRYPOINT=worker is the worker.
	// (See TestIntegration for how this happens.)
	//
	// Unfortunately this can't be a true Example because of the skip,
	// which requires a testing.T.
	func TestExampleSeparateAnalysis(t *testing.T) {
	testenv.NeedsGoPackages(t)

	// src is an archive containing a module with a printf mistake.
	const src = `
	-- go.mod --
	module separate
	go 1.18

	-- main/main.go --
	package main

	import "separate/lib"

	func main() {
	lib.MyPrintf("%s", 123)
	}

	-- lib/lib.go --
	package lib

	import "fmt"

	func MyPrintf(format string, args ...any) {
	fmt.Printf(format, args...)
	}
	`

	// Expand archive into tmp tree.
	fs, err := txtar.FS(txtar.Parse([]byte(src)))
	if err != nil {
	t.Fatal(err)
	}
	tmpdir := testfiles.CopyToTmp(t, fs)

	// Load metadata for the main package and all its dependencies.
	cfg := &packages.Config{
	Mode: packages.NeedName \| packages.NeedFiles \| packages.NeedCompiledGoFiles \| packages.NeedImports \| packages.NeedModule,
	Dir: tmpdir,
	Env: append(os.Environ(),
	"GOPROXY=off", // disable network
	"GOWORK=off", // an ambient GOWORK value would break package loading
	),
	Logf: t.Logf,
	}
	pkgs, err := packages.Load(cfg, "separate/main")
	if err != nil {
	t.Fatal(err)
	}
	// Stop if any package had a metadata error.
	if packages.PrintErrors(pkgs) > 0 {
	t.Fatal("there were errors among loaded packages")
	}

	// Now we have loaded the import graph,
	// let's begin the proper work of the manager.

	// Gather root packages. They will get all analyzers,
	// whereas dependencies get only the subset that
	// produce facts or are required by them.
	roots := make(map[*packages.Package]bool)
	for _, pkg := range pkgs {
	roots[pkg] = true
	}

	// nextID generates sequence numbers for each unit of work.
	// We use it to create names of temporary files.
	var nextID atomic.Int32

	var allDiagnostics []string

	// Visit all packages in postorder: dependencies first.
	// TODO(adonovan): opt: use parallel postorder.
	packages.Visit(pkgs, nil, func(pkg *packages.Package) {
	if pkg.PkgPath == "unsafe" {
	return
	}

	// Choose a unique prefix for temporary files
	// (.cfg .types .facts) produced by this package.
	// We stow it in an otherwise unused field of
	// Package so it can be accessed by our importers.
	prefix := fmt.Sprintf("%s/%d", tmpdir, nextID.Add(1))
	pkg.ExportFile = prefix

	// Construct the request to the worker.
	var (
	importMap = make(map[string]string)
	packageFile = make(map[string]string)
	packageVetx = make(map[string]string)
	)
	for importPath, dep := range pkg.Imports {
	importMap[importPath] = dep.PkgPath
	if depPrefix := dep.ExportFile; depPrefix != "" { // skip "unsafe"
	packageFile[dep.PkgPath] = depPrefix + ".types"
	packageVetx[dep.PkgPath] = depPrefix + ".facts"
	}
	}
	cfg := unitchecker.Config{
	ID: pkg.ID,
	ImportPath: pkg.PkgPath,
	GoFiles: pkg.CompiledGoFiles,
	NonGoFiles: pkg.OtherFiles,
	IgnoredFiles: pkg.IgnoredFiles,
	ImportMap: importMap,
	PackageFile: packageFile,
	PackageVetx: packageVetx,
	VetxOnly: !roots[pkg],
	VetxOutput: prefix + ".facts",
	}
	if pkg.Module != nil {
	if v := pkg.Module.GoVersion; v != "" {
	cfg.GoVersion = "go" + v
	}
	cfg.ModulePath = pkg.Module.Path
	cfg.ModuleVersion = pkg.Module.Version
	}

	// Write the JSON configuration message to a file.
	cfgData, err := json.Marshal(cfg)
	if err != nil {
	t.Fatalf("internal error in json.Marshal: %v", err)
	}
	cfgFile := prefix + ".cfg"
	if err := os.WriteFile(cfgFile, cfgData, 0666); err != nil {
	t.Fatal(err)
	}

	// Send the request to the worker.
	cmd := testenv.Command(t, os.Args[0], "-json", cfgFile)
	cmd.Stderr = os.Stderr
	cmd.Stdout = new(bytes.Buffer)
	cmd.Env = append(os.Environ(), "ENTRYPOINT=worker")
	if err := cmd.Run(); err != nil {
	t.Fatal(err)
	}

	// Parse JSON output and gather in allDiagnostics.
	dec := json.NewDecoder(cmd.Stdout.(io.Reader))
	for {
	type jsonDiagnostic struct {
	Posn string `json:"posn"`
	Message string `json:"message"`
	}
	// 'results' maps Package.Path -> Analyzer.Name -> diagnostics
	var results map[string]map[string][]jsonDiagnostic
	if err := dec.Decode(&results); err != nil {
	if err == io.EOF {
	break
	}
	t.Fatalf("internal error decoding JSON: %v", err)
	}
	for _, result := range results {
	for analyzer, diags := range result {
	for _, diag := range diags {
	rel := strings.ReplaceAll(diag.Posn, tmpdir, "")
	rel = filepath.ToSlash(rel)
	msg := fmt.Sprintf("%s: [%s] %s", rel, analyzer, diag.Message)
	allDiagnostics = append(allDiagnostics, msg)
	}
	}
	}
	}
	})

	// Observe that the example produces a fact-based diagnostic
	// from separate analysis of "main", "lib", and "fmt":

	const want = `/main/main.go:6:2: [printf] separate/lib.MyPrintf format %s has arg 123 of wrong type int`
	sort.Strings(allDiagnostics)
	if got := strings.Join(allDiagnostics, "\n"); got != want {
	t.Errorf("Got: %s\nWant: %s", got, want)
	}
	}

	// -- worker process --

	// worker is the main entry point for a unitchecker-based driver
	// with only a single analyzer, for illustration.
	func worker() {
	// Currently the unitchecker API doesn't allow clients to
	// control exactly how and where fact and type information
	// is produced and consumed.
	//
	// So, for example, it assumes that type information has
	// already been produced by the compiler, which is true when
	// running under "go vet", but isn't necessary. It may be more
	// convenient and efficient for a distributed analysis system
	// if the worker generates both of them, which is the approach
	// taken in this example; they could even be saved as two
	// sections of a single file.
	//
	// Consequently, this test currently needs special access to
	// private hooks in unitchecker to control how and where facts
	// and types are produced and consumed. In due course this
	// will become a respectable public API. In the meantime, it
	// should at least serve as a demonstration of how one could
	// fork unitchecker to achieve separate analysis without go vet.
	unitchecker.SetTypeImportExport(makeTypesImporter, exportTypes)

	unitchecker.Main(printf.Analyzer)
	}

	func makeTypesImporter(cfg unitchecker.Config, fset token.FileSet) types.Importer {
	imports := make(map[string]*types.Package)
	return importerFunc(func(importPath string) (*types.Package, error) {
	// Resolve import path to package path (vendoring, etc)
	path, ok := cfg.ImportMap[importPath]
	if !ok {
	return nil, fmt.Errorf("can't resolve import %q", path)
	}
	if path == "unsafe" {
	return types.Unsafe, nil
	}

	// Find, read, and decode file containing type information.
	file, ok := cfg.PackageFile[path]
	if !ok {
	return nil, fmt.Errorf("no package file for %q", path)
	}
	f, err := os.Open(file)
	if err != nil {
	return nil, err
	}
	defer f.Close() // ignore error
	return gcexportdata.Read(f, fset, imports, path)
	})
	}

	func exportTypes(cfg unitchecker.Config, fset token.FileSet, pkg *types.Package) error {
	var out bytes.Buffer
	if err := gcexportdata.Write(&out, fset, pkg); err != nil {
	return err
	}
	typesFile := strings.TrimSuffix(cfg.VetxOutput, ".facts") + ".types"
	return os.WriteFile(typesFile, out.Bytes(), 0666)
	}

	// -- helpers --

	type importerFunc func(path string) (*types.Package, error)

	func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) }