cmd/go: use cache for PGO preprocessing
This is the final CL in the series adding PGO preprocessing support to
cmd/go. Now that the tool is hooked up, we integrate with the build
cache to cache the result.
This is fairly straightforward. One difference is that the compile and
link do caching through updateBuildID. However, preprocessed PGO files
don't have a build ID, so it doesn't make much sense to hack our way
through that function when it is simple to just add to the cache
ourselves.
As as aside, we could add a build ID to the preproccessed file format,
though it is not clear if it is worthwhile. The one place a build ID
could be used is in buildActionID, which currently compute the file hash
of the preprocessed profile. With a build ID it could simply read the
build ID. This would save one complete read of the file per build
(cmd/go caches the hash), but each compile process also reads the entire
file, so this is a small change overall.
Fixes #58102.
Cq-Include-Trybots: luci.golang.try:gotip-linux-amd64-longtest
Change-Id: I86e2999a08ccd264230fbb1c983192259b7288e9
Reviewed-on: https://go-review.googlesource.com/c/go/+/569425
Auto-Submit: Michael Pratt <mpratt@google.com>
Reviewed-by: Cherry Mui <cherryyz@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
diff --git a/doc/next/5-toolchain.md b/doc/next/5-toolchain.md
index 0f4a816..ce763f1 100644
--- a/doc/next/5-toolchain.md
+++ b/doc/next/5-toolchain.md
@@ -1,5 +1,9 @@
## Compiler {#compiler}
+The build time overhead to building with [Profile Guided Optimization](/doc/pgo) has been reduced significantly.
+Previously, large builds could see 100%+ build time increase from enabling PGO.
+In Go 1.23, overhead should be in the single digit percentages.
+
## Assembler {#assembler}
## Linker {#linker}
diff --git a/src/cmd/go/internal/work/action.go b/src/cmd/go/internal/work/action.go
index 249c802..5e83f1e 100644
--- a/src/cmd/go/internal/work/action.go
+++ b/src/cmd/go/internal/work/action.go
@@ -461,6 +461,17 @@
return b.build(ctx, a)
}
+// pgoActionID computes the action ID for a preprocess PGO action.
+func (b *Builder) pgoActionID(input string) cache.ActionID {
+ h := cache.NewHash("preprocess PGO profile " + input)
+
+ fmt.Fprintf(h, "preprocess PGO profile\n")
+ fmt.Fprintf(h, "preprofile %s\n", b.toolID("preprofile"))
+ fmt.Fprintf(h, "input %q\n", b.fileHash(input))
+
+ return h.Sum()
+}
+
// pgoActor implements the Actor interface for preprocessing PGO profiles.
type pgoActor struct {
// input is the path to the original pprof profile.
@@ -468,7 +479,10 @@
}
func (p *pgoActor) Act(b *Builder, ctx context.Context, a *Action) error {
- // TODO(prattmic): Integrate with build cache to cache output.
+ if b.useCache(a, b.pgoActionID(p.input), a.Target, !b.IsCmdList) || b.IsCmdList {
+ return nil
+ }
+ defer b.flushOutput(a)
sh := b.Shell(a)
@@ -480,7 +494,33 @@
return err
}
+ // N.B. Builder.build looks for the out in a.built, regardless of
+ // whether this came from cache.
a.built = a.Target
+
+ if !cfg.BuildN {
+ // Cache the output.
+ //
+ // N.B. We don't use updateBuildID here, as preprocessed PGO profiles
+ // do not contain a build ID. updateBuildID is typically responsible
+ // for adding to the cache, thus we must do so ourselves instead.
+
+ r, err := os.Open(a.Target)
+ if err != nil {
+ return fmt.Errorf("error opening target for caching: %w", err)
+ }
+
+ c := cache.Default()
+ outputID, _, err := c.Put(a.actionID, r)
+ r.Close()
+ if err != nil {
+ return fmt.Errorf("error adding target to cache: %w", err)
+ }
+ if cfg.BuildX {
+ sh.ShowCmd("", "%s # internal", joinUnambiguously(str.StringList("cp", a.Target, c.OutputFile(outputID))))
+ }
+ }
+
return nil
}
diff --git a/src/cmd/go/internal/work/buildid.go b/src/cmd/go/internal/work/buildid.go
index bf923d0..acbda1a 100644
--- a/src/cmd/go/internal/work/buildid.go
+++ b/src/cmd/go/internal/work/buildid.go
@@ -527,6 +527,14 @@
// Check to see if the action output is cached.
if file, _, err := cache.GetFile(c, actionHash); err == nil {
+ if a.Mode == "preprocess PGO profile" {
+ // Preprocessed PGO profiles don't embed a build ID, so
+ // skip the build ID lookup.
+ // TODO(prattmic): better would be to add a build ID to the format.
+ a.built = file
+ a.Target = "DO NOT USE - using cache"
+ return true
+ }
if buildID, err := buildid.ReadFile(file); err == nil {
if printOutput {
showStdout(b, c, a, "stdout")
diff --git a/src/cmd/go/testdata/script/build_cache_pgo.txt b/src/cmd/go/testdata/script/build_cache_pgo.txt
new file mode 100644
index 0000000..5efecab
--- /dev/null
+++ b/src/cmd/go/testdata/script/build_cache_pgo.txt
@@ -0,0 +1,28 @@
+[short] skip
+
+# Set up fresh GOCACHE.
+env GOCACHE=$WORK/gocache
+mkdir $GOCACHE
+
+# Building trivial non-main package should run preprofile the first time.
+go build -x -pgo=default.pgo lib.go
+stderr 'preprofile.*default\.pgo'
+
+# ... but not again ...
+go build -x -pgo=default.pgo lib.go
+! stderr 'preprofile.*default\.pgo'
+
+# ... unless we use -a.
+go build -a -x -pgo=default.pgo lib.go
+stderr 'preprofile.*default\.pgo'
+
+# ... building a different package should not run preprofile again, instead using a profile from cache.
+go build -x -pgo=default.pgo lib2.go
+! stderr 'preprofile.*default\.pgo'
+stderr 'compile.*-pgoprofile=.*'$GOCACHE'.*lib2.go'
+
+-- lib.go --
+package lib
+-- lib2.go --
+package lib2
+-- default.pgo --
diff --git a/src/cmd/go/testdata/script/build_pgo.txt b/src/cmd/go/testdata/script/build_pgo.txt
index 0ca2105..792d299 100644
--- a/src/cmd/go/testdata/script/build_pgo.txt
+++ b/src/cmd/go/testdata/script/build_pgo.txt
@@ -3,6 +3,10 @@
[short] skip 'compiles and links executables'
+# Set up fresh GOCACHE.
+env GOCACHE=$WORK/gocache
+mkdir $GOCACHE
+
# build without PGO
go build triv.go
diff --git a/src/cmd/go/testdata/script/build_pgo_auto.txt b/src/cmd/go/testdata/script/build_pgo_auto.txt
index 1ae86d4..dc25702 100644
--- a/src/cmd/go/testdata/script/build_pgo_auto.txt
+++ b/src/cmd/go/testdata/script/build_pgo_auto.txt
@@ -2,6 +2,10 @@
[short] skip 'compiles and links executables'
+# Set up fresh GOCACHE.
+env GOCACHE=$WORK/gocache
+mkdir $GOCACHE
+
# use default.pgo for a single main package
go build -n -pgo=auto -o a1.exe ./a/a1
stderr 'preprofile.*-i.*default\.pgo'
diff --git a/src/cmd/preprofile/main.go b/src/cmd/preprofile/main.go
index 4cb87f6..f29b527 100644
--- a/src/cmd/preprofile/main.go
+++ b/src/cmd/preprofile/main.go
@@ -16,6 +16,7 @@
import (
"bufio"
+ "cmd/internal/objabi"
"cmd/internal/pgo"
"flag"
"fmt"
@@ -67,6 +68,8 @@
}
func main() {
+ objabi.AddVersionFlag()
+
log.SetFlags(0)
log.SetPrefix("preprofile: ")
