src/cmd/compile/internal/arm64/pair.go

// Copyright 2026 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package arm64

import (
	"cmd/compile/internal/base"
	"cmd/compile/internal/objw"
	"cmd/internal/obj"
	"cmd/internal/obj/arm64"
	"cmd/internal/src"
)

// movdImmMemInfo describes a plain "MOVD reg, off(base)" or "MOVD off(base), reg"
// instruction. base, off, name, and sym record the memory operand's base
// register, offset, addressing class (NAME_AUTO or NAME_PARAM), and symbol;
// reg is the register stored or loaded, and isLoad distinguishes the two
// forms.
type movdImmMemInfo struct {
	base, reg int16
	off       int64
	name      obj.AddrName
	sym       *obj.LSym
	isLoad    bool
}

// movdImmMem decodes p into a movdImmMemInfo. ok reports whether p is a
// plain MOVD load or store using an immediate-offset addressing form
// suitable for LDP/STP coalescing.
func movdImmMem(p *obj.Prog) (info movdImmMemInfo, ok bool) {
	if p.As != arm64.AMOVD || p.Scond != 0 || p.Reg != 0 {
		return movdImmMemInfo{}, false
	}
	var a *obj.Addr
	switch {
	case p.From.Type == obj.TYPE_MEM && p.To.Type == obj.TYPE_REG:
		// Plain load: MOVD mem, reg.
		a = &p.From
		info.reg = p.To.Reg
		info.isLoad = true
	case p.From.Type == obj.TYPE_REG && p.To.Type == obj.TYPE_MEM:
		// Plain store: MOVD reg, mem.
		a = &p.To
		info.reg = p.From.Reg
	default:
		return movdImmMemInfo{}, false
	}
	if a.Index != 0 || (a.Name != obj.NAME_AUTO && a.Name != obj.NAME_PARAM) {
		return movdImmMemInfo{}, false
	}
	info.base, info.off, info.name, info.sym = a.Reg, a.Offset, a.Name, a.Sym
	return info, true
}

// ssaGenFinish runs after genssa has emitted all of a function's Progs,
// resolved its branch and jump-table targets, and finalized the frame size
// in defframe.
func ssaGenFinish(pp *objw.Progs) {
	if base.Flag.N != 0 {
		// Keep unoptimized builds unoptimized.
		return
	}
	pairSpills(pp.Text, pp.Text.To.Offset, pp.CurFunc.LSym.Func().JumpTables)
}

// pairSpills fuses strictly-adjacent MOVD load or store pairs that target the
// same base register at consecutive 8-byte offsets into a single LDP or STP.
// text is the function's first Prog, framesize its final frame size, and
// jumpTables its resolved jump tables.
//
// This recovers spill/reload pairings that the SSA-level pair pass
// (cmd/compile/internal/ssa/pair.go) cannot perform: that pass runs before
// regalloc and only sees source-level loads/stores, so the STR/LDR pairs that
// regalloc later inserts for OpStoreReg/OpLoadReg never get a chance to be
// paired. Doing the fusion here, as the last step of code generation, keeps the
// optimization in the compiler so the assembler can stay a simple translator.
func pairSpills(text *obj.Prog, framesize int64, jumpTables []obj.JumpTable) {
	// Branch and jump-table targets, collected lazily once a candidate pair
	// survives the cheaper checks; most functions have no candidates.
	// Fusing into the first of a pair whose second instruction is a target
	// would silently change control flow: paths that jump directly to the
	// second instruction would skip the work the LDP/STP now does at the
	// first instruction's position.
	var isTarget map[*obj.Prog]bool
	targets := func() map[*obj.Prog]bool {
		if isTarget == nil {
			isTarget = map[*obj.Prog]bool{}
			for p := text; p != nil; p = p.Link {
				if t := p.To.Target(); t != nil {
					isTarget[t] = true
				}
			}
			for _, jt := range jumpTables {
				for _, t := range jt.Targets {
					isTarget[t] = true
				}
			}
		}
		return isTarget
	}
	for p := text; p != nil; p = p.Link {
		q := p.Link
		if q == nil {
			continue
		}
		a, ok := movdImmMem(p)
		if !ok {
			continue
		}
		b, ok := movdImmMem(q)
		if !ok || a.isLoad != b.isLoad {
			continue
		}
		if a.base != b.base || a.name != b.name {
			continue
		}
		if a.reg == b.reg {
			// LDP with Rt1 == Rt2 is CONSTRAINED UNPREDICTABLE. (STP with
			// identical source registers would be fine, but spills never
			// store the same register twice, so don't bother.)
			continue
		}
		if a.isLoad && a.reg == a.base {
			// The first load overwrites the base register: executed
			// sequentially, the second load computes its address from the
			// just-loaded value, while LDP computes both addresses from
			// the original base. (The second load overwriting the base is
			// fine; no address depends on it.)
			continue
		}
		if q.Pos.IsStmt() == src.PosIsStmt {
			// q carries a statement boundary, and may also be registered
			// as an inline mark: genssa promotes instructions it reuses
			// as inline marks to statements. Reducing q to a 0-byte ANOP
			// would drop the statement from the line tables and violate
			// the invariant that inline marks are never zero-sized (they
			// are identified by PC).
			continue
		}
		lo, hi := a, b
		if lo.off > hi.off {
			lo, hi = hi, lo
		}
		if hi.off != lo.off+8 {
			continue
		}
		// Fuse only when the result will encode as a single LDP/STP, whose
		// signed 7-bit immediate scaled by 8 covers offsets [-512, 504].
		// The assembler resolves a NAME_AUTO offset to off+framesize+8
		// (the 8 is the saved LR) and a NAME_PARAM offset to
		// off+framesize+24 or +32 depending on frame alignment padding —
		// or off+8 in a frameless leaf, which is not decided until
		// assembly, so a frameless PARAM must fit both. An out-of-range
		// LDP/STP needs an assembler-synthesized address (ADD + LDP),
		// which is no smaller than the original pair and serializes the
		// accesses through REGTMP.
		var biasLo, biasHi int64
		switch {
		case lo.name == obj.NAME_AUTO:
			// Autos exist only in functions with a frame, and their offsets
			// are negative from FP, so framesize+8 rebases them onto the
			// non-negative SP-relative range the LDP/STP immediate must reach.
			biasLo = framesize + 8
			biasHi = biasLo
		case framesize == 0:
			biasLo, biasHi = 8, 24
		case framesize%16 == 0:
			biasLo = framesize + 24
			biasHi = biasLo
		default:
			biasLo = framesize + 32
			biasHi = biasLo
		}
		if lo.off%8 != 0 || lo.off+biasLo < -512 || lo.off+biasHi > 504 {
			continue
		}
		if targets()[q] {
			continue
		}
		mem := obj.Addr{Type: obj.TYPE_MEM, Reg: lo.base, Offset: lo.off, Name: lo.name, Sym: lo.sym}
		regs := obj.Addr{Type: obj.TYPE_REGREG, Reg: lo.reg, Offset: int64(hi.reg)}
		if a.isLoad {
			p.As = arm64.ALDP
			p.From, p.To = mem, regs
		} else {
			p.As = arm64.ASTP
			p.From, p.To = regs, mem
		}
		// Reduce q to a 0-byte ANOP rather than unlinking it so that
		// branch targets referencing q remain valid.
		obj.Nopout(q)
	}
}