| // Copyright 2023 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| // Package interleaved implements the interleaved devirtualization and |
| // inlining pass. |
| package interleaved |
| |
| import ( |
| "cmd/compile/internal/base" |
| "cmd/compile/internal/devirtualize" |
| "cmd/compile/internal/inline" |
| "cmd/compile/internal/inline/inlheur" |
| "cmd/compile/internal/ir" |
| "cmd/compile/internal/pgo" |
| "cmd/compile/internal/typecheck" |
| "fmt" |
| ) |
| |
| // DevirtualizeAndInlinePackage interleaves devirtualization and inlining on |
| // all functions within pkg. |
| func DevirtualizeAndInlinePackage(pkg *ir.Package, profile *pgo.Profile) { |
| if profile != nil && base.Debug.PGODevirtualize > 0 { |
| // TODO(mdempsky): Integrate into DevirtualizeAndInlineFunc below. |
| ir.VisitFuncsBottomUp(typecheck.Target.Funcs, func(list []*ir.Func, recursive bool) { |
| for _, fn := range list { |
| devirtualize.ProfileGuided(fn, profile) |
| } |
| }) |
| ir.CurFunc = nil |
| } |
| |
| if base.Flag.LowerL != 0 { |
| inlheur.SetupScoreAdjustments() |
| } |
| |
| var inlProfile *pgo.Profile // copy of profile for inlining |
| if base.Debug.PGOInline != 0 { |
| inlProfile = profile |
| } |
| if inlProfile != nil { |
| inline.PGOInlinePrologue(inlProfile, pkg.Funcs) |
| } |
| |
| ir.VisitFuncsBottomUp(pkg.Funcs, func(funcs []*ir.Func, recursive bool) { |
| // We visit functions within an SCC in fairly arbitrary order, |
| // so by computing inlinability for all functions in the SCC |
| // before performing any inlining, the results are less |
| // sensitive to the order within the SCC (see #58905 for an |
| // example). |
| |
| // First compute inlinability for all functions in the SCC ... |
| inline.CanInlineSCC(funcs, recursive, inlProfile) |
| |
| // ... then make a second pass to do devirtualization and inlining |
| // of calls. |
| for _, fn := range funcs { |
| DevirtualizeAndInlineFunc(fn, inlProfile) |
| } |
| }) |
| |
| if base.Flag.LowerL != 0 { |
| // Perform a garbage collection of hidden closures functions that |
| // are no longer reachable from top-level functions following |
| // inlining. See #59404 and #59638 for more context. |
| inline.GarbageCollectUnreferencedHiddenClosures() |
| |
| if base.Debug.DumpInlFuncProps != "" { |
| inlheur.DumpFuncProps(nil, base.Debug.DumpInlFuncProps) |
| } |
| if inlheur.Enabled() { |
| inline.PostProcessCallSites(inlProfile) |
| inlheur.TearDown() |
| } |
| } |
| } |
| |
| // DevirtualizeAndInlineFunc interleaves devirtualization and inlining |
| // on a single function. |
| func DevirtualizeAndInlineFunc(fn *ir.Func, profile *pgo.Profile) { |
| ir.WithFunc(fn, func() { |
| if base.Flag.LowerL != 0 { |
| if inlheur.Enabled() && !fn.Wrapper() { |
| inlheur.ScoreCalls(fn) |
| defer inlheur.ScoreCallsCleanup() |
| } |
| if base.Debug.DumpInlFuncProps != "" && !fn.Wrapper() { |
| inlheur.DumpFuncProps(fn, base.Debug.DumpInlFuncProps) |
| } |
| } |
| |
| bigCaller := base.Flag.LowerL != 0 && inline.IsBigFunc(fn) |
| if bigCaller && base.Flag.LowerM > 1 { |
| fmt.Printf("%v: function %v considered 'big'; reducing max cost of inlinees\n", ir.Line(fn), fn) |
| } |
| |
| // Walk fn's body and apply devirtualization and inlining. |
| var inlCalls []*ir.InlinedCallExpr |
| var edit func(ir.Node) ir.Node |
| edit = func(n ir.Node) ir.Node { |
| switch n := n.(type) { |
| case *ir.TailCallStmt: |
| n.Call.NoInline = true // can't inline yet |
| } |
| |
| ir.EditChildren(n, edit) |
| |
| if call, ok := n.(*ir.CallExpr); ok { |
| devirtualize.StaticCall(call) |
| |
| if inlCall := inline.TryInlineCall(fn, call, bigCaller, profile); inlCall != nil { |
| inlCalls = append(inlCalls, inlCall) |
| n = inlCall |
| } |
| } |
| |
| return n |
| } |
| ir.EditChildren(fn, edit) |
| |
| // If we inlined any calls, we want to recursively visit their |
| // bodies for further devirtualization and inlining. However, we |
| // need to wait until *after* the original function body has been |
| // expanded, or else inlCallee can have false positives (e.g., |
| // #54632). |
| for len(inlCalls) > 0 { |
| call := inlCalls[0] |
| inlCalls = inlCalls[1:] |
| ir.EditChildren(call, edit) |
| } |
| }) |
| } |