cmd/compile: preload multi-instruction constants before loops on arm64
The register allocator never preloads rematerializeable values into
registers before a loop, so a 64-bit constant that needs MOVD plus up
to three MOVKs on arm64 is rebuilt on every iteration. Preload such
constants when a register is free, as is already done for other
values live into the loop.
Single-instruction constants are unchanged.
Neoverse N1, repro from the issue (HashVar loads the constant from a
variable and is unaffected):
│ base │ new │
│ sec/op │ sec/op vs base │
Hash 5.165µ ± 0% 4.128µ ± 0% -20.09% (p=0.000 n=10)
HashVar 4.128µ ± 0% 4.129µ ± 0% ~ (p=0.778 n=10)
For #81755
Change-Id: I87c5779b24057cd59833c2afc32c7ddef06a29df
Reviewed-on: https://go-review.googlesource.com/c/go/+/840405
Reviewed-by: Keith Randall <khr@google.com>
LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Auto-Submit: Keith Randall <khr@golang.org>
Reviewed-by: David Chase <drchase@google.com>
Reviewed-by: Keith Randall <khr@golang.org>
diff --git a/src/cmd/compile/internal/ssacompile/regalloc.go b/src/cmd/compile/internal/ssacompile/regalloc.go
index 73af23f..47102c0 100644
--- a/src/cmd/compile/internal/ssacompile/regalloc.go
+++ b/src/cmd/compile/internal/ssacompile/regalloc.go
@@ -157,6 +157,25 @@
return bits.OnesCount64(r.V1) + bits.OnesCount64(r.V2)
}
+// costlyRemat reports whether rematerializing v takes more than one
+// instruction.
+// TODO: logical immediates (bitcon) are also one instruction on arm64.
+func costlyRemat(v *ssa.Value) bool {
+ if v.Op != ssaop.OpARM64MOVDconst {
+ return false
+ }
+ zeros, ones := 0, 0
+ for i := 0; i < 64; i += 16 {
+ switch uint16(v.AuxInt >> i) {
+ case 0:
+ zeros++
+ case 0xffff:
+ ones++
+ }
+ }
+ return zeros < 3 && ones < 3
+}
+
// pickReg picks a register from the register mask.
func (s *regAllocState) pickReg(rm ssaop.RegMask) ssaop.Register {
if s.f.Config.Ctxt.Arch.Arch == sys.ArchRISCV64 {
@@ -2014,14 +2033,15 @@
if !vi.Regs.Empty() {
continue
}
- if vi.Rematerializeable {
- // TODO: maybe we should not skip rematerializeable
- // values here. One rematerialization outside the loop
- // is better than N in the loop. But rematerializations
- // are cheap, and spilling another value may not be.
- // And we don't want to materialize the zero register
- // into a different register when it is just the
- // argument to a store.
+ if vi.Rematerializeable && !costlyRemat(v) {
+ // Most rematerializations are a single instruction,
+ // and spilling another value to make room is not
+ // worth it. Nor do we want to materialize the zero
+ // register into a different register when it is just
+ // the argument to a store. Multi-instruction
+ // constants are worth building once outside the loop
+ // if a register is free (m below excludes registers
+ // holding likely-used values).
continue
}
}
diff --git a/src/cmd/compile/internal/ssacompile/regalloc_test.go b/src/cmd/compile/internal/ssacompile/regalloc_test.go
index ace4fb6..f31ba9b 100644
--- a/src/cmd/compile/internal/ssacompile/regalloc_test.go
+++ b/src/cmd/compile/internal/ssacompile/regalloc_test.go
@@ -410,6 +410,65 @@
}
}
+// TestPreloadCostlyConstant checks that a constant needing several
+// instructions to build is loaded before a loop rather than rebuilt inside it,
+// while single-instruction constants are still rematerialized at their use.
+func TestPreloadCostlyConstant(t *testing.T) {
+ for _, tc := range []struct {
+ k int64
+ inLoop bool
+ }{
+ {k: 7, inLoop: true}, // MOVD
+ {k: -0x1234, inLoop: true}, // MOVN
+ {k: -0x30e44323485a9b9d, inLoop: false}, // MOVD + 3×MOVK (0xcf1bbcdcb7a56463)
+ {k: 0x12340000_5678, inLoop: false}, // MOVD + MOVK
+ } {
+ t.Run(fmt.Sprintf("%#x", uint64(tc.k)), func(t *testing.T) {
+ c := testConfigARM64(t)
+ i64 := c.config.Types.Int64
+ f := c.Fun("entry",
+ Bloc("entry",
+ Valu("mem", ssaop.OpInitMem, types.TypeMem, 0, nil),
+ Valu("addr", ssaop.OpArg, i64.PtrTo(), 0, c.Temp(i64.PtrTo())),
+ Valu("k", ssaop.OpARM64MOVDconst, i64, tc.k, nil),
+ Valu("init", ssaop.OpARM64MOVDconst, i64, 1, nil),
+ Goto("loopHead"),
+ ),
+ Bloc("loopHead",
+ Valu("i", ssaop.OpPhi, i64, 0, nil, "init", "next"),
+ Valu("acc", ssaop.OpPhi, i64, 0, nil, "init", "prod"),
+ Valu("test", ssaop.OpARM64CMPconst, types.TypeFlags, 10, nil, "i"),
+ Eq("test", "exit", "loopBody"),
+ ),
+ Bloc("loopBody",
+ Valu("next", ssaop.OpARM64ADDconst, i64, 1, nil, "i"),
+ Valu("prod", ssaop.OpARM64MUL, i64, 0, nil, "acc", "k"),
+ Goto("loopHead"),
+ ),
+ Bloc("exit",
+ Valu("store", ssaop.OpARM64MOVDstore, types.TypeMem, 0, nil, "addr", "acc", "mem"),
+ Exit("store"),
+ ),
+ )
+ f.f.Blocks[1].Likely = ssa.BranchUnlikely
+ regalloc(f.f)
+ checkFunc(f.f)
+
+ n := 0
+ for _, b := range []string{"loopHead", "loopBody"} {
+ for _, v := range f.blocks[b].Values {
+ if v.Op == ssaop.OpARM64MOVDconst && v.AuxInt == tc.k {
+ n++
+ }
+ }
+ }
+ if got := n > 0; got != tc.inLoop {
+ t.Errorf("constant materialized in loop = %v, want %v:\n%v", got, tc.inLoop, f.f)
+ }
+ })
+ }
+}
+
// TestStartRegsDrop tests dropping physical register bits from startRegs preventing
// unnecessary OpLoadReg on edges, such as edge right->merge in this example.
//
