cmd/updatestd/updatestd.go - build - Git at Google

 // Copyright 2020 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 // updatestd is an experimental program that has been used to update
 // the standard library modules as part of golang.org/issue/36905 in
 // CL 255860 and CL 266898. It's expected to be modified to meet the
 // ongoing needs of that recurring maintenance work.
 package main

 import (
 	"bytes"
 	"context"
 	"debug/buildinfo"
 	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
 	"go/ast"
 	"go/parser"
 	"go/token"
 	"io"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strconv"
 	"strings"

 	"golang.org/x/build/gerrit"
 	"golang.org/x/build/internal/envutil"
 )

 var goCmd string // the go command

 func main() {
 	log.SetFlags(0)

 	flag.Usage = func() {
 		fmt.Fprintln(os.Stderr, "Usage: updatestd -goroot=<goroot> -branch=<branch>")
 		flag.PrintDefaults()
 	}
 	goroot := flag.String("goroot", "", "path to a working copy of https://go.googlesource.com/go (required)")
 	branch := flag.String("branch", "", "branch to target, such as master or release-branch.go1.Y (required)")
 	flag.Parse()
 	if flag.NArg() != 0 || *goroot == "" || *branch == "" {
 		flag.Usage()
 		os.Exit(2)
 	}

 	// Determine the Go version from the GOROOT source tree.
 	goVersion, err := gorootVersion(*goroot)
 	if err != nil {
 		log.Fatalln(err)
 	}

 	goCmd = filepath.Join(*goroot, "bin", "go")

 	// Confirm that bundle is in PATH.
 	// It's needed for a go generate step later.
 	bundlePath, err := exec.LookPath("bundle")
 	if err != nil {
 		log.Fatalln("can't find bundle in PATH; did you run 'go install golang.org/x/tools/cmd/bundle@latest' and add it to PATH?")
 	}
 	if bi, err := buildinfo.ReadFile(bundlePath); err != nil || bi.Path != "golang.org/x/tools/cmd/bundle" {
 		// Not the bundle command we want.
 		log.Fatalln("unexpected bundle command in PATH; did you run 'go install golang.org/x/tools/cmd/bundle@latest' and add it to PATH?")
 	}

 	// Fetch latest hashes of Go projects from Gerrit,
 	// using the specified branch name.
 	//
 	// This gives us a consistent snapshot of all golang.org/x module versions
 	// at a given point in time. This ensures selection of latest available
 	// pseudo-versions is done without being subject to module mirror caching,
 	// and that selected pseudo-versions can be re-used across multiple modules.
 	//
 	// TODO: Consider a future enhancement of fetching build status for all
 	// commits that are selected and reporting if any of them have a failure.
 	//
 	cl := gerrit.NewClient("https://go-review.googlesource.com", gerrit.NoAuth)
 	projs, err := cl.GetProjects(context.Background(), *branch)
 	if err != nil {
 		log.Fatalln("failed to get a list of Gerrit projects:", err)
 	}
 	hashes := map[string]string{}
 	for name, p := range projs {
 		if p.State != "ACTIVE" {
 			continue
 		}
 		if hash, ok := p.Branches[*branch]; ok {
 			hashes[name] = hash
 		}
 	}

 	w := Work{
 		Branch:        *branch,
 		GoVersion:     fmt.Sprintf("1.%d", goVersion),
 		ProjectHashes: hashes,
 	}

 	// Print environment information.
 	r := runner{filepath.Join(*goroot, "src")}
 	r.run(goCmd, "version")
 	r.run(goCmd, "env", "GOROOT")
 	r.run(goCmd, "version", "-m", bundlePath)
 	log.Println()

 	// Walk the standard library source tree (GOROOT/src),
 	// skipping directories that the Go command ignores (see go help packages)
 	// and update modules that are found.
 	err = filepath.Walk(filepath.Join(*goroot, "src"), func(path string, fi os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 		if fi.IsDir() && (strings.HasPrefix(fi.Name(), ".") || strings.HasPrefix(fi.Name(), "_") || fi.Name() == "testdata" || fi.Name() == "vendor") {
 			return filepath.SkipDir
 		}
 		goModFile := fi.Name() == "go.mod" && !fi.IsDir()
 		if goModFile {
 			moduleDir := filepath.Dir(path)
 			err := w.UpdateModule(moduleDir)
 			if err != nil {
 				return fmt.Errorf("failed to update module in %s: %v", moduleDir, err)
 			}
 			return filepath.SkipDir // Skip the remaining files in this directory.
 		}
 		return nil
 	})
 	if err != nil {
 		log.Fatalln(err)
 	}

 	// Re-bundle packages in the standard library.
 	//
 	// TODO: Maybe do GOBIN=$(mktemp -d) go install golang.org/x/tools/cmd/bundle@version or so,
 	// and add it to PATH to eliminate variance in bundle tool version. Can be considered later.
 	//
 	log.Println("updating bundles in", r.dir)
 	r.run(goCmd, "generate", "-run=bundle", "std", "cmd")
 }

 type Work struct {
 	Branch        string            // Target branch name.
 	GoVersion     string            // Major Go version, like "1.x".
 	ProjectHashes map[string]string // Gerrit project name → commit hash.
 }

 // UpdateModule updates the standard library module found in dir:
 //
 //  1. Set the expected Go version in go.mod file to w.GoVersion.
 //  2. For modules in the build list with "golang.org/x/" prefix,
 //     update to pseudo-version corresponding to w.ProjectHashes.
 //  3. Run go mod tidy.
 //  4. Run go mod vendor.
 //
 // The logic in this method needs to serve the dependency update
 // policy for the purpose of golang.org/issue/36905, although it
 // does not directly define said policy.
 func (w Work) UpdateModule(dir string) error {
 	// Determine the build list.
 	main, deps := buildList(dir)

 	// Determine module versions to get.
 	goGet := []string{goCmd, "get", "-d"}
 	for _, m := range deps {
 		if !strings.HasPrefix(m.Path, "golang.org/x/") {
 			log.Printf("skipping %s (out of scope, it's not a golang.org/x dependency)\n", m.Path)
 			continue
 		}
 		gerritProj := m.Path[len("golang.org/x/"):]
 		hash, ok := w.ProjectHashes[gerritProj]
 		if !ok {
 			if m.Indirect {
 				log.Printf("skipping %s because branch %s doesn't exist and it's indirect\n", m.Path, w.Branch)
 				continue
 			}
 			return fmt.Errorf("no hash for Gerrit project %q", gerritProj)
 		}
 		goGet = append(goGet, m.Path+"@"+hash)
 	}

 	// Run all the commands.
 	log.Println("updating module", main.Path, "in", dir)
 	r := runner{dir}
 	gowork := strings.TrimSpace(string(r.runOut(goCmd, "env", "GOWORK")))
 	if gowork != "" && gowork != "off" {
 		log.Printf("warning: GOWORK=%q, things may go wrong?", gowork)
 	}
 	r.run(goCmd, "mod", "edit", "-go="+w.GoVersion)
 	r.run(goGet...)
 	r.run(goCmd, "mod", "tidy")
 	r.run(goCmd, "mod", "vendor")
 	log.Println()
 	return nil
 }

 // buildList determines the build list in the directory dir
 // by invoking the go command. It uses -mod=readonly mode.
 // It returns the main module and other modules separately
 // for convenience to the UpdateModule caller.
 //
 // See https://golang.org/cmd/go/#hdr-The_main_module_and_the_build_list
 // and https://golang.org/ref/mod#glos-build-list.
 func buildList(dir string) (main module, deps []module) {
 	out := runner{dir}.runOut(goCmd, "list", "-mod=readonly", "-m", "-json", "all")
 	for dec := json.NewDecoder(bytes.NewReader(out)); ; {
 		var m module
 		err := dec.Decode(&m)
 		if err == io.EOF {
 			break
 		} else if err != nil {
 			log.Fatalf("internal error: unexpected problem decoding JSON returned by go list -json: %v", err)
 		}
 		if m.Main {
 			main = m
 			continue
 		}
 		deps = append(deps, m)
 	}
 	return main, deps
 }

 type module struct {
 	Path     string // Module path.
 	Main     bool   // Is this the main module?
 	Indirect bool   // Is this module only an indirect dependency of main module?
 }

 // gorootVersion reads the GOROOT/src/internal/goversion/goversion.go
 // file and reports the Version declaration value found therein.
 func gorootVersion(goroot string) (int, error) {
 	// Parse the goversion.go file, extract the declaration from the AST.
 	//
 	// This is a pragmatic approach that relies on the trajectory of the
 	// internal/goversion package being predictable and unlikely to change.
 	// If that stops being true, this small helper is easy to re-write.
 	//
 	fset := token.NewFileSet()
 	f, err := parser.ParseFile(fset, filepath.Join(goroot, "src", "internal", "goversion", "goversion.go"), nil, 0)
 	if os.IsNotExist(err) {
 		return 0, fmt.Errorf("did not find goversion.go file (%v); wrong goroot or did internal/goversion package change?", err)
 	} else if err != nil {
 		return 0, err
 	}
 	for _, d := range f.Decls {
 		g, ok := d.(*ast.GenDecl)
 		if !ok {
 			continue
 		}
 		for _, s := range g.Specs {
 			v, ok := s.(*ast.ValueSpec)
 			if !ok || len(v.Names) != 1 || v.Names[0].String() != "Version" || len(v.Values) != 1 {
 				continue
 			}
 			l, ok := v.Values[0].(*ast.BasicLit)
 			if !ok || l.Kind != token.INT {
 				continue
 			}
 			return strconv.Atoi(l.Value)
 		}
 	}
 	return 0, fmt.Errorf("did not find Version declaration in %s; wrong goroot or did internal/goversion package change?", fset.File(f.Pos()).Name())
 }

 type runner struct{ dir string }

 // run runs the command and requires that it succeeds.
 // It logs the command's combined output.
 func (r runner) run(args ...string) {
 	log.Printf("> %s\n", strings.Join(args, " "))
 	cmd := exec.Command(args[0], args[1:]...)
 	envutil.SetDir(cmd, r.dir)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		log.Fatalf("command failed: %s\n%s", err, out)
 	}
 	if len(out) != 0 {
 		log.Print(string(out))
 	}
 }

 // runOut runs the command, requires that it succeeds,
 // and returns the command's standard output.
 func (r runner) runOut(args ...string) []byte {
 	cmd := exec.Command(args[0], args[1:]...)
 	envutil.SetDir(cmd, r.dir)
 	out, err := cmd.Output()
 	if err != nil {
 		log.Printf("> %s\n", strings.Join(args, " "))
 		if ee := (*exec.ExitError)(nil); errors.As(err, &ee) {
 			out = append(out, ee.Stderr...)
 		}
 		log.Fatalf("command failed: %s\n%s", err, out)
 	}
 	return out
 }
	// Copyright 2020 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	// updatestd is an experimental program that has been used to update
	// the standard library modules as part of golang.org/issue/36905 in
	// CL 255860 and CL 266898. It's expected to be modified to meet the
	// ongoing needs of that recurring maintenance work.
	package main

	import (
	"bytes"
	"context"
	"debug/buildinfo"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"golang.org/x/build/gerrit"
	"golang.org/x/build/internal/envutil"
	)

	var goCmd string // the go command

	func main() {
	log.SetFlags(0)

	flag.Usage = func() {
	fmt.Fprintln(os.Stderr, "Usage: updatestd -goroot=<goroot> -branch=<branch>")
	flag.PrintDefaults()
	}
	goroot := flag.String("goroot", "", "path to a working copy of https://go.googlesource.com/go (required)")
	branch := flag.String("branch", "", "branch to target, such as master or release-branch.go1.Y (required)")
	flag.Parse()
	if flag.NArg() != 0 \|\| goroot == "" \|\| branch == "" {
	flag.Usage()
	os.Exit(2)
	}

	// Determine the Go version from the GOROOT source tree.
	goVersion, err := gorootVersion(*goroot)
	if err != nil {
	log.Fatalln(err)
	}

	goCmd = filepath.Join(*goroot, "bin", "go")

	// Confirm that bundle is in PATH.
	// It's needed for a go generate step later.
	bundlePath, err := exec.LookPath("bundle")
	if err != nil {
	log.Fatalln("can't find bundle in PATH; did you run 'go install golang.org/x/tools/cmd/bundle@latest' and add it to PATH?")
	}
	if bi, err := buildinfo.ReadFile(bundlePath); err != nil \|\| bi.Path != "golang.org/x/tools/cmd/bundle" {
	// Not the bundle command we want.
	log.Fatalln("unexpected bundle command in PATH; did you run 'go install golang.org/x/tools/cmd/bundle@latest' and add it to PATH?")
	}

	// Fetch latest hashes of Go projects from Gerrit,
	// using the specified branch name.
	//
	// This gives us a consistent snapshot of all golang.org/x module versions
	// at a given point in time. This ensures selection of latest available
	// pseudo-versions is done without being subject to module mirror caching,
	// and that selected pseudo-versions can be re-used across multiple modules.
	//
	// TODO: Consider a future enhancement of fetching build status for all
	// commits that are selected and reporting if any of them have a failure.
	//
	cl := gerrit.NewClient("https://go-review.googlesource.com", gerrit.NoAuth)
	projs, err := cl.GetProjects(context.Background(), *branch)
	if err != nil {
	log.Fatalln("failed to get a list of Gerrit projects:", err)
	}
	hashes := map[string]string{}
	for name, p := range projs {
	if p.State != "ACTIVE" {
	continue
	}
	if hash, ok := p.Branches[*branch]; ok {
	hashes[name] = hash
	}
	}

	w := Work{
	Branch: *branch,
	GoVersion: fmt.Sprintf("1.%d", goVersion),
	ProjectHashes: hashes,
	}

	// Print environment information.
	r := runner{filepath.Join(*goroot, "src")}
	r.run(goCmd, "version")
	r.run(goCmd, "env", "GOROOT")
	r.run(goCmd, "version", "-m", bundlePath)
	log.Println()

	// Walk the standard library source tree (GOROOT/src),
	// skipping directories that the Go command ignores (see go help packages)
	// and update modules that are found.
	err = filepath.Walk(filepath.Join(*goroot, "src"), func(path string, fi os.FileInfo, err error) error {
	if err != nil {
	return err
	}
	if fi.IsDir() && (strings.HasPrefix(fi.Name(), ".") \|\| strings.HasPrefix(fi.Name(), "_") \|\| fi.Name() == "testdata" \|\| fi.Name() == "vendor") {
	return filepath.SkipDir
	}
	goModFile := fi.Name() == "go.mod" && !fi.IsDir()
	if goModFile {
	moduleDir := filepath.Dir(path)
	err := w.UpdateModule(moduleDir)
	if err != nil {
	return fmt.Errorf("failed to update module in %s: %v", moduleDir, err)
	}
	return filepath.SkipDir // Skip the remaining files in this directory.
	}
	return nil
	})
	if err != nil {
	log.Fatalln(err)
	}

	// Re-bundle packages in the standard library.
	//
	// TODO: Maybe do GOBIN=$(mktemp -d) go install golang.org/x/tools/cmd/bundle@version or so,
	// and add it to PATH to eliminate variance in bundle tool version. Can be considered later.
	//
	log.Println("updating bundles in", r.dir)
	r.run(goCmd, "generate", "-run=bundle", "std", "cmd")
	}

	type Work struct {
	Branch string // Target branch name.
	GoVersion string // Major Go version, like "1.x".
	ProjectHashes map[string]string // Gerrit project name → commit hash.
	}

	// UpdateModule updates the standard library module found in dir:
	//
	// 1. Set the expected Go version in go.mod file to w.GoVersion.
	// 2. For modules in the build list with "golang.org/x/" prefix,
	// update to pseudo-version corresponding to w.ProjectHashes.
	// 3. Run go mod tidy.
	// 4. Run go mod vendor.
	//
	// The logic in this method needs to serve the dependency update
	// policy for the purpose of golang.org/issue/36905, although it
	// does not directly define said policy.
	func (w Work) UpdateModule(dir string) error {
	// Determine the build list.
	main, deps := buildList(dir)

	// Determine module versions to get.
	goGet := []string{goCmd, "get", "-d"}
	for _, m := range deps {
	if !strings.HasPrefix(m.Path, "golang.org/x/") {
	log.Printf("skipping %s (out of scope, it's not a golang.org/x dependency)\n", m.Path)
	continue
	}
	gerritProj := m.Path[len("golang.org/x/"):]
	hash, ok := w.ProjectHashes[gerritProj]
	if !ok {
	if m.Indirect {
	log.Printf("skipping %s because branch %s doesn't exist and it's indirect\n", m.Path, w.Branch)
	continue
	}
	return fmt.Errorf("no hash for Gerrit project %q", gerritProj)
	}
	goGet = append(goGet, m.Path+"@"+hash)
	}

	// Run all the commands.
	log.Println("updating module", main.Path, "in", dir)
	r := runner{dir}
	gowork := strings.TrimSpace(string(r.runOut(goCmd, "env", "GOWORK")))
	if gowork != "" && gowork != "off" {
	log.Printf("warning: GOWORK=%q, things may go wrong?", gowork)
	}
	r.run(goCmd, "mod", "edit", "-go="+w.GoVersion)
	r.run(goGet...)
	r.run(goCmd, "mod", "tidy")
	r.run(goCmd, "mod", "vendor")
	log.Println()
	return nil
	}

	// buildList determines the build list in the directory dir
	// by invoking the go command. It uses -mod=readonly mode.
	// It returns the main module and other modules separately
	// for convenience to the UpdateModule caller.
	//
	// See https://golang.org/cmd/go/#hdr-The_main_module_and_the_build_list
	// and https://golang.org/ref/mod#glos-build-list.
	func buildList(dir string) (main module, deps []module) {
	out := runner{dir}.runOut(goCmd, "list", "-mod=readonly", "-m", "-json", "all")
	for dec := json.NewDecoder(bytes.NewReader(out)); ; {
	var m module
	err := dec.Decode(&m)
	if err == io.EOF {
	break
	} else if err != nil {
	log.Fatalf("internal error: unexpected problem decoding JSON returned by go list -json: %v", err)
	}
	if m.Main {
	main = m
	continue
	}
	deps = append(deps, m)
	}
	return main, deps
	}

	type module struct {
	Path string // Module path.
	Main bool // Is this the main module?
	Indirect bool // Is this module only an indirect dependency of main module?
	}

	// gorootVersion reads the GOROOT/src/internal/goversion/goversion.go
	// file and reports the Version declaration value found therein.
	func gorootVersion(goroot string) (int, error) {
	// Parse the goversion.go file, extract the declaration from the AST.
	//
	// This is a pragmatic approach that relies on the trajectory of the
	// internal/goversion package being predictable and unlikely to change.
	// If that stops being true, this small helper is easy to re-write.
	//
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, filepath.Join(goroot, "src", "internal", "goversion", "goversion.go"), nil, 0)
	if os.IsNotExist(err) {
	return 0, fmt.Errorf("did not find goversion.go file (%v); wrong goroot or did internal/goversion package change?", err)
	} else if err != nil {
	return 0, err
	}
	for _, d := range f.Decls {
	g, ok := d.(*ast.GenDecl)
	if !ok {
	continue
	}
	for _, s := range g.Specs {
	v, ok := s.(*ast.ValueSpec)
	if !ok \|\| len(v.Names) != 1 \|\| v.Names[0].String() != "Version" \|\| len(v.Values) != 1 {
	continue
	}
	l, ok := v.Values[0].(*ast.BasicLit)
	if !ok \|\| l.Kind != token.INT {
	continue
	}
	return strconv.Atoi(l.Value)
	}
	}
	return 0, fmt.Errorf("did not find Version declaration in %s; wrong goroot or did internal/goversion package change?", fset.File(f.Pos()).Name())
	}

	type runner struct{ dir string }

	// run runs the command and requires that it succeeds.
	// It logs the command's combined output.
	func (r runner) run(args ...string) {
	log.Printf("> %s\n", strings.Join(args, " "))
	cmd := exec.Command(args[0], args[1:]...)
	envutil.SetDir(cmd, r.dir)
	out, err := cmd.CombinedOutput()
	if err != nil {
	log.Fatalf("command failed: %s\n%s", err, out)
	}
	if len(out) != 0 {
	log.Print(string(out))
	}
	}

	// runOut runs the command, requires that it succeeds,
	// and returns the command's standard output.
	func (r runner) runOut(args ...string) []byte {
	cmd := exec.Command(args[0], args[1:]...)
	envutil.SetDir(cmd, r.dir)
	out, err := cmd.Output()
	if err != nil {
	log.Printf("> %s\n", strings.Join(args, " "))
	if ee := (*exec.ExitError)(nil); errors.As(err, &ee) {
	out = append(out, ee.Stderr...)
	}
	log.Fatalf("command failed: %s\n%s", err, out)
	}
	return out
	}