src/cmd/compile/internal/ssacompile/expand_calls.go

// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ssacompile

import (
	"fmt"

	"cmd/compile/internal/abi"
	"cmd/compile/internal/base"
	"cmd/compile/internal/ir"
	"cmd/compile/internal/ssa"
	"cmd/compile/internal/ssa/block"
	"cmd/compile/internal/ssa/ssaop"
	"cmd/compile/internal/types"
	"cmd/internal/src"
)

func postExpandCallsDecompose(f *ssa.Func) {
	decomposeUser(f)    // redo user decompose to cleanup after expand calls
	decomposeBuiltin(f) // handles both regular decomposition and cleanup.
}

func expandCalls(f *ssa.Func) {
	// Convert each aggregate arg to a call into "dismantle aggregate, store/pass parts"
	// Convert each aggregate result from a call into "assemble aggregate from parts"
	// Convert each multivalue exit into "dismantle aggregate, store/return parts"
	// Convert incoming aggregate arg into assembly of parts.
	// Feed modified AST to decompose.

	sp, _ := f.SpSb()

	x := &expandState{
		f:               f,
		debug:           f.Pass.Debug,
		regSize:         f.Config.RegSize,
		sp:              sp,
		typs:            &f.Config.Types,
		wideSelects:     make(map[*ssa.Value]*ssa.Value),
		commonArgs:      make(map[selKey]*ssa.Value),
		commonSelectors: make(map[selKey]*ssa.Value),
		memForCall:      make(map[ssa.ID]*ssa.Value),
	}

	// For 32-bit, need to deal with decomposition of 64-bit integers, which depends on endianness.
	if f.Config.BigEndian {
		x.firstOp = ssaop.OpInt64Hi
		x.secondOp = ssaop.OpInt64Lo
		x.firstType = x.typs.Int32
		x.secondType = x.typs.UInt32
	} else {
		x.firstOp = ssaop.OpInt64Lo
		x.secondOp = ssaop.OpInt64Hi
		x.firstType = x.typs.UInt32
		x.secondType = x.typs.Int32
	}

	// Defer select processing until after all calls and selects are seen.
	var selects []*ssa.Value
	var calls []*ssa.Value
	var args []*ssa.Value
	var exitBlocks []*ssa.Block

	var m0 *ssa.Value

	// Accumulate lists of calls, args, selects, and exit blocks to process,
	// note "wide" selects consumed by stores,
	// rewrite mem for each call,
	// rewrite each OpSelectNAddr.
	for _, b := range f.Blocks {
		for _, v := range b.Values {
			switch v.Op {
			case ssaop.OpInitMem:
				m0 = v

			case ssaop.OpClosureLECall, ssaop.OpInterLECall, ssaop.OpStaticLECall, ssaop.OpTailLECall, ssaop.OpTailLECallInter:
				calls = append(calls, v)

			case ssaop.OpArg:
				args = append(args, v)

			case ssaop.OpStore:
				if a := v.Args[1]; a.Op == ssaop.OpSelectN && !ssa.CanSSA(a.Type) {
					if a.Uses > 1 {
						panic(fmt.Errorf("Saw double use of wide SelectN %s operand of Store %s",
							a.LongString(), v.LongString()))
					}
					x.wideSelects[a] = v
				}

			case ssaop.OpSelectN:
				if v.Type == types.TypeMem {
					// rewrite the mem selector in place
					call := v.Args[0]
					aux := call.Aux.(*ssa.AuxCall)
					mem := x.memForCall[call.ID]
					if mem == nil {
						v.AuxInt = int64(aux.AbiInfo.OutRegistersUsed())
						x.memForCall[call.ID] = v
					} else {
						panic(fmt.Errorf("Saw two memories for call %v, %v and %v", call, mem, v))
					}
				} else {
					selects = append(selects, v)
				}

			case ssaop.OpSelectNAddr:
				call := v.Args[0]
				which := v.AuxInt
				aux := call.Aux.(*ssa.AuxCall)
				pt := v.Type
				off := x.offsetFrom(x.f.Entry, x.sp, aux.OffsetOfResult(which), pt)
				v.CopyOf(off)
			}
		}

		// rewrite function results from an exit block
		// values returned by function need to be split out into registers.
		if isBlockMultiValueExit(b) {
			exitBlocks = append(exitBlocks, b)
		}
	}

	// Convert each aggregate arg into Make of its parts (and so on, to primitive types)
	for _, v := range args {
		var rc registerCursor
		a := x.prAssignForArg(v)
		aux := x.f.OwnAux
		regs := a.Registers
		var offset int64
		if len(regs) == 0 {
			offset = a.FrameOffset(aux.AbiInfo)
		}
		auxBase := x.offsetFrom(x.f.Entry, x.sp, offset, types.NewPtr(v.Type))
		rc.init(regs, aux.AbiInfo, nil, auxBase, 0)
		x.rewriteSelectOrArg(f.Entry.Pos, f.Entry, v, v, m0, v.Type, rc)
	}

	// Rewrite selects of results (which may be aggregates) into make-aggregates of register/memory-targeted selects
	for _, v := range selects {
		if v.Op == ssaop.OpInvalid {
			continue
		}

		call := v.Args[0]
		aux := call.Aux.(*ssa.AuxCall)
		mem := x.memForCall[call.ID]
		if mem == nil {
			mem = call.Block.NewValue1I(call.Pos, ssaop.OpSelectN, types.TypeMem, int64(aux.AbiInfo.OutRegistersUsed()), call)
			x.memForCall[call.ID] = mem
		}

		i := v.AuxInt
		regs := aux.RegsOfResult(i)

		// If this select cannot fit into SSA and is stored, either disaggregate to register stores, or mem-mem move.
		if store := x.wideSelects[v]; store != nil {
			// Use the mem that comes from the store operation.
			storeAddr := store.Args[0]
			mem := store.Args[2]
			if len(regs) > 0 {
				// Cannot do a rewrite that builds up a result from pieces; instead, copy pieces to the store operation.
				var rc registerCursor
				rc.init(regs, aux.AbiInfo, nil, storeAddr, 0)
				mem = x.rewriteWideSelectToStores(call.Pos, call.Block, v, mem, v.Type, rc)
				store.CopyOf(mem)
			} else {
				// Move directly from AuxBase to store target; rewrite the store instruction.
				offset := aux.OffsetOfResult(i)
				auxBase := x.offsetFrom(x.f.Entry, x.sp, offset, types.NewPtr(v.Type))
				// was Store dst, v, mem
				// now Move dst, auxBase, mem
				move := store.Block.NewValue3A(store.Pos, ssaop.OpMove, types.TypeMem, v.Type, storeAddr, auxBase, mem)
				move.AuxInt = v.Type.Size()
				store.CopyOf(move)
			}
			continue
		}

		var auxBase *ssa.Value
		if len(regs) == 0 {
			offset := aux.OffsetOfResult(i)
			auxBase = x.offsetFrom(x.f.Entry, x.sp, offset, types.NewPtr(v.Type))
		}
		var rc registerCursor
		rc.init(regs, aux.AbiInfo, nil, auxBase, 0)
		x.rewriteSelectOrArg(call.Pos, call.Block, v, v, mem, v.Type, rc)
	}

	rewriteCall := func(v *ssa.Value, newOp ssaop.Op, argStart int) {
		// Break aggregate args passed to call into smaller pieces.
		x.rewriteCallArgs(v, argStart)
		v.Op = newOp
		rts := abi.RegisterTypes(v.Aux.(*ssa.AuxCall).AbiInfo.OutParams())
		v.Type = types.NewResults(append(rts, types.TypeMem))
	}

	// Rewrite calls
	for _, v := range calls {
		switch v.Op {
		case ssaop.OpStaticLECall:
			rewriteCall(v, ssaop.OpStaticCall, 0)
		case ssaop.OpTailLECall:
			rewriteCall(v, ssaop.OpTailCall, 0)
		case ssaop.OpTailLECallInter:
			rewriteCall(v, ssaop.OpTailCallInter, 1)
		case ssaop.OpClosureLECall:
			rewriteCall(v, ssaop.OpClosureCall, 2)
		case ssaop.OpInterLECall:
			rewriteCall(v, ssaop.OpInterCall, 1)
		}
	}

	// Rewrite results from exit blocks
	for _, b := range exitBlocks {
		v := b.Controls[0]
		x.rewriteFuncResults(v, b, f.OwnAux)
		b.SetControl(v)
	}

}

func (x *expandState) rewriteFuncResults(v *ssa.Value, b *ssa.Block, aux *ssa.AuxCall) {
	// This is very similar to rewriteCallArgs
	// differences:
	// firstArg + preArgs
	// sp vs auxBase

	m0 := v.MemoryArg()
	mem := m0

	allResults := []*ssa.Value{}
	var oldArgs []*ssa.Value
	argsWithoutMem := v.Args[:len(v.Args)-1]

	for j, a := range argsWithoutMem {
		oldArgs = append(oldArgs, a)
		i := int64(j)
		auxType := aux.TypeOfResult(i)
		auxBase := b.NewValue2A(v.Pos, ssaop.OpLocalAddr, types.NewPtr(auxType), aux.NameOfResult(i), x.sp, mem)
		auxOffset := int64(0)
		aRegs := aux.RegsOfResult(int64(j))
		if a.Op == ssaop.OpDereference {
			a.Op = ssaop.OpLoad
		}
		var rc registerCursor
		var result *[]*ssa.Value
		if len(aRegs) > 0 {
			result = &allResults
		} else {
			if a.Op == ssaop.OpLoad && a.Args[0].Op == ssaop.OpLocalAddr && a.Args[0].Aux == aux.NameOfResult(i) {
				continue // Self move to output parameter
			}
		}
		rc.init(aRegs, aux.AbiInfo, result, auxBase, auxOffset)
		mem = x.decomposeAsNecessary(v.Pos, b, a, mem, rc)
	}
	v.ResetArgs()
	v.AddArgs(allResults...)
	v.AddArg(mem)
	for _, a := range oldArgs {
		if a.Uses == 0 {
			if x.debug > 1 {
				x.Printf("...marking %v unused\n", a.LongString())
			}
			x.invalidateRecursively(a)
		}
	}
	v.Type = types.NewResults(append(abi.RegisterTypes(aux.AbiInfo.OutParams()), types.TypeMem))
	return
}

func (x *expandState) rewriteCallArgs(v *ssa.Value, firstArg int) {
	if x.debug > 1 {
		x.indent(3)
		defer x.indent(-3)
		x.Printf("rewriteCallArgs(%s; %d)\n", v.LongString(), firstArg)
	}
	// Thread the stores on the memory arg
	aux := v.Aux.(*ssa.AuxCall)
	m0 := v.MemoryArg()
	mem := m0
	allResults := []*ssa.Value{}
	oldArgs := []*ssa.Value{}
	argsWithoutMem := v.Args[firstArg : len(v.Args)-1] // Also strip closure/interface Op-specific args

	sp := x.sp
	if v.Op == ssaop.OpTailLECall || v.Op == ssaop.OpTailLECallInter {
		// For tail call, we unwind the frame before the call so we'll use the caller's
		// SP.
		sp = v.Block.NewValue1(src.NoXPos, ssaop.OpGetCallerSP, x.typs.Uintptr, mem)
	}

	for i, a := range argsWithoutMem { // skip leading non-parameter SSA Args and trailing mem SSA Arg.
		oldArgs = append(oldArgs, a)
		auxI := int64(i)
		aRegs := aux.RegsOfArg(auxI)
		aType := aux.TypeOfArg(auxI)

		if a.Op == ssaop.OpDereference {
			a.Op = ssaop.OpLoad
		}
		var rc registerCursor
		var result *[]*ssa.Value
		var aOffset int64
		if len(aRegs) > 0 {
			result = &allResults
		} else {
			aOffset = aux.OffsetOfArg(auxI)
		}
		if v.Op == ssaop.OpTailLECall && a.Op == ssaop.OpArg && a.AuxInt == 0 {
			// It's common for a tail call passing the same arguments (e.g. method wrapper),
			// so this would be a self copy. Detect this and optimize it out.
			n := a.Aux.(*ir.Name)
			if n.Class == ir.PPARAM && n.FrameOffset()+x.f.Config.Ctxt.Arch.FixedFrameSize == aOffset {
				continue
			}
		}
		if x.debug > 1 {
			x.Printf("...storeArg %s, %v, %d\n", a.LongString(), aType, aOffset)
		}

		rc.init(aRegs, aux.AbiInfo, result, sp, aOffset)
		mem = x.decomposeAsNecessary(v.Pos, v.Block, a, mem, rc)
	}
	var preArgStore [2]*ssa.Value
	preArgs := append(preArgStore[:0], v.Args[0:firstArg]...)
	v.ResetArgs()
	v.AddArgs(preArgs...)
	v.AddArgs(allResults...)
	v.AddArg(mem)
	for _, a := range oldArgs {
		if a.Uses == 0 {
			x.invalidateRecursively(a)
		}
	}

	return
}

func (x *expandState) decomposePair(pos src.XPos, b *ssa.Block, a, mem *ssa.Value, t0, t1 *types.Type, o0, o1 ssaop.Op, rc *registerCursor) *ssa.Value {
	e := b.NewValue1(pos, o0, t0, a)
	pos = pos.WithNotStmt()
	mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(t0))
	e = b.NewValue1(pos, o1, t1, a)
	mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(t1))
	return mem
}

func (x *expandState) decomposeOne(pos src.XPos, b *ssa.Block, a, mem *ssa.Value, t0 *types.Type, o0 ssaop.Op, rc *registerCursor) *ssa.Value {
	e := b.NewValue1(pos, o0, t0, a)
	pos = pos.WithNotStmt()
	mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(t0))
	return mem
}

// decomposeAsNecessary converts a value (perhaps an aggregate) passed to a call or returned by a function,
// into the appropriate sequence of stores and register assignments to transmit that value in a given ABI, and
// returns the current memory after this convert/rewrite (it may be the input memory, perhaps stores were needed.)
// 'pos' is the source position all this is tied to
// 'b' is the enclosing block
// 'a' is the value to decompose
// 'm0' is the input memory arg used for the first store (or returned if there are no stores)
// 'rc' is a registerCursor which identifies the register/memory destination for the value
func (x *expandState) decomposeAsNecessary(pos src.XPos, b *ssa.Block, a, m0 *ssa.Value, rc registerCursor) *ssa.Value {
	if x.debug > 1 {
		x.indent(3)
		defer x.indent(-3)
	}
	at := a.Type
	if at.Size() == 0 {
		return m0
	}
	if a.Op == ssaop.OpDereference {
		a.Op = ssaop.OpLoad // For purposes of parameter passing expansion, a Dereference is a Load.
	}

	if !rc.hasRegs() && !ssa.CanSSA(at) {
		dst := x.offsetFrom(b, rc.storeDest, rc.storeOffset, types.NewPtr(at))
		if x.debug > 1 {
			x.Printf("...recur store %s at %s\n", a.LongString(), dst.LongString())
		}
		if a.Op == ssaop.OpLoad {
			m0 = b.NewValue3A(pos, ssaop.OpMove, types.TypeMem, at, dst, a.Args[0], m0)
			m0.AuxInt = at.Size()
			return m0
		} else {
			panic(fmt.Errorf("Store of not a load"))
		}
	}

	mem := m0
	switch at.Kind() {
	case types.TARRAY:
		et := at.Elem()
		for i := int64(0); i < at.NumElem(); i++ {
			e := b.NewValue1I(pos, ssaop.OpArraySelect, et, i, a)
			pos = pos.WithNotStmt()
			mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(et))
		}
		return mem

	case types.TSTRUCT:
		if at.IsSIMD() {
			break // XXX
		}
		for i := 0; i < at.NumFields(); i++ {
			et := at.Field(i).Type // might need to read offsets from the fields
			e := b.NewValue1I(pos, ssaop.OpStructSelect, et, int64(i), a)
			pos = pos.WithNotStmt()
			if x.debug > 1 {
				x.Printf("...recur decompose %s, %v\n", e.LongString(), et)
			}
			mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(et))
		}
		return mem

	case types.TSLICE:
		mem = x.decomposeOne(pos, b, a, mem, at.Elem().PtrTo(), ssaop.OpSlicePtr, &rc)
		pos = pos.WithNotStmt()
		mem = x.decomposeOne(pos, b, a, mem, x.typs.Int, ssaop.OpSliceLen, &rc)
		return x.decomposeOne(pos, b, a, mem, x.typs.Int, ssaop.OpSliceCap, &rc)

	case types.TSTRING:
		return x.decomposePair(pos, b, a, mem, x.typs.BytePtr, x.typs.Int, ssaop.OpStringPtr, ssaop.OpStringLen, &rc)

	case types.TINTER:
		mem = x.decomposeOne(pos, b, a, mem, x.typs.Uintptr, ssaop.OpITab, &rc)
		pos = pos.WithNotStmt()
		// Immediate interfaces cause so many headaches.
		if a.Op == ssaop.OpIMake {
			data := a.Args[1]
			for data.Op == ssaop.OpStructMake || data.Op == ssaop.OpArrayMake1 {
				// A struct make might have a few zero-sized fields.
				// Use the pointer-y one we know is there.
				for _, a := range data.Args {
					if a.Type.Size() > 0 {
						data = a
						break
					}
				}
			}
			return x.decomposeAsNecessary(pos, b, data, mem, rc.next(data.Type))
		}
		return x.decomposeOne(pos, b, a, mem, x.typs.BytePtr, ssaop.OpIData, &rc)

	case types.TCOMPLEX64:
		return x.decomposePair(pos, b, a, mem, x.typs.Float32, x.typs.Float32, ssaop.OpComplexReal, ssaop.OpComplexImag, &rc)

	case types.TCOMPLEX128:
		return x.decomposePair(pos, b, a, mem, x.typs.Float64, x.typs.Float64, ssaop.OpComplexReal, ssaop.OpComplexImag, &rc)

	case types.TINT64:
		if at.Size() > x.regSize {
			return x.decomposePair(pos, b, a, mem, x.firstType, x.secondType, x.firstOp, x.secondOp, &rc)
		}
	case types.TUINT64:
		if at.Size() > x.regSize {
			return x.decomposePair(pos, b, a, mem, x.typs.UInt32, x.typs.UInt32, x.firstOp, x.secondOp, &rc)
		}
	}

	// An atomic type, either record the register or store it and update the memory.

	if rc.hasRegs() {
		if x.debug > 1 {
			x.Printf("...recur addArg %s\n", a.LongString())
		}
		rc.addArg(a)
	} else {
		dst := x.offsetFrom(b, rc.storeDest, rc.storeOffset, types.NewPtr(at))
		if x.debug > 1 {
			x.Printf("...recur store %s at %s\n", a.LongString(), dst.LongString())
		}
		mem = b.NewValue3A(pos, ssaop.OpStore, types.TypeMem, at, dst, a, mem)
	}

	return mem
}

// Convert scalar OpArg into the proper OpWhateverArg instruction
// Convert scalar OpSelectN into perhaps-differently-indexed OpSelectN
// Convert aggregate OpArg into Make of its parts (which are eventually scalars)
// Convert aggregate OpSelectN into Make of its parts (which are eventually scalars)
// Returns the converted value.
//
//   - "pos" the position for any generated instructions
//   - "b" the block for any generated instructions
//   - "container" the outermost OpArg/OpSelectN
//   - "a" the instruction to overwrite, if any (only the outermost caller)
//   - "m0" the memory arg for any loads that are necessary
//   - "at" the type of the Arg/part
//   - "rc" the register/memory cursor locating the various parts of the Arg.
func (x *expandState) rewriteSelectOrArg(pos src.XPos, b *ssa.Block, container, a, m0 *ssa.Value, at *types.Type, rc registerCursor) *ssa.Value {

	if at == types.TypeMem {
		a.CopyOf(m0)
		return a
	}

	makeOf := func(a *ssa.Value, op ssaop.Op, args []*ssa.Value) *ssa.Value {
		if a == nil {
			a = b.NewValue0(pos, op, at)
			a.AddArgs(args...)
		} else {
			a.ResetArgs()
			a.Aux, a.AuxInt = nil, 0
			a.Pos, a.Op, a.Type = pos, op, at
			a.AddArgs(args...)
		}
		return a
	}

	if at.Size() == 0 {
		// For consistency, create these values even though they'll ultimately be unused
		return makeOf(a, ssaop.OpEmpty, nil)
	}

	sk := selKey{from: container, size: 0, offsetOrIndex: rc.storeOffset, typ: at}
	dupe := x.commonSelectors[sk]
	if dupe != nil {
		if a == nil {
			return dupe
		}
		a.CopyOf(dupe)
		return a
	}

	var argStore [10]*ssa.Value
	args := argStore[:0]

	addArg := func(a0 *ssa.Value) {
		if a0 == nil {
			as := "<nil>"
			if a != nil {
				as = a.LongString()
			}
			panic(fmt.Errorf("a0 should not be nil, a=%v, container=%v, at=%v", as, container.LongString(), at))
		}
		args = append(args, a0)
	}

	switch at.Kind() {
	case types.TARRAY:
		et := at.Elem()
		for i := int64(0); i < at.NumElem(); i++ {
			e := x.rewriteSelectOrArg(pos, b, container, nil, m0, et, rc.next(et))
			addArg(e)
		}
		a = makeOf(a, ssaop.OpArrayMake1, args)
		x.commonSelectors[sk] = a
		return a

	case types.TSTRUCT:
		// Assume ssagen/ssa.go (in buildssa) spills large aggregates so they won't appear here.
		if at.IsSIMD() {
			break // XXX
		}
		for i := 0; i < at.NumFields(); i++ {
			et := at.Field(i).Type
			e := x.rewriteSelectOrArg(pos, b, container, nil, m0, et, rc.next(et))
			if e == nil {
				panic(fmt.Errorf("nil e, et=%v, et.Size()=%d, i=%d", et, et.Size(), i))
			}
			addArg(e)
			pos = pos.WithNotStmt()
		}
		if at.NumFields() > ssa.MaxStruct && !types.IsDirectIface(at) {
			panic(fmt.Errorf("Too many fields (%d, %d bytes), container=%s", at.NumFields(), at.Size(), container.LongString()))
		}
		a = makeOf(a, ssaop.OpStructMake, args)
		x.commonSelectors[sk] = a
		return a

	case types.TSLICE:
		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, at.Elem().PtrTo(), rc.next(x.typs.BytePtr)))
		pos = pos.WithNotStmt()
		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Int, rc.next(x.typs.Int)))
		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Int, rc.next(x.typs.Int)))
		a = makeOf(a, ssaop.OpSliceMake, args)
		x.commonSelectors[sk] = a
		return a

	case types.TSTRING:
		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.BytePtr, rc.next(x.typs.BytePtr)))
		pos = pos.WithNotStmt()
		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Int, rc.next(x.typs.Int)))
		a = makeOf(a, ssaop.OpStringMake, args)
		x.commonSelectors[sk] = a
		return a

	case types.TINTER:
		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Uintptr, rc.next(x.typs.Uintptr)))
		pos = pos.WithNotStmt()
		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.BytePtr, rc.next(x.typs.BytePtr)))
		a = makeOf(a, ssaop.OpIMake, args)
		x.commonSelectors[sk] = a
		return a

	case types.TCOMPLEX64:
		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Float32, rc.next(x.typs.Float32)))
		pos = pos.WithNotStmt()
		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Float32, rc.next(x.typs.Float32)))
		a = makeOf(a, ssaop.OpComplexMake, args)
		x.commonSelectors[sk] = a
		return a

	case types.TCOMPLEX128:
		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Float64, rc.next(x.typs.Float64)))
		pos = pos.WithNotStmt()
		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Float64, rc.next(x.typs.Float64)))
		a = makeOf(a, ssaop.OpComplexMake, args)
		x.commonSelectors[sk] = a
		return a

	case types.TINT64:
		if at.Size() > x.regSize {
			addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.firstType, rc.next(x.firstType)))
			pos = pos.WithNotStmt()
			addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.secondType, rc.next(x.secondType)))
			if !x.f.Config.BigEndian {
				// Int64Make args are big, little
				args[0], args[1] = args[1], args[0]
			}
			a = makeOf(a, ssaop.OpInt64Make, args)
			x.commonSelectors[sk] = a
			return a
		}
	case types.TUINT64:
		if at.Size() > x.regSize {
			addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.UInt32, rc.next(x.typs.UInt32)))
			pos = pos.WithNotStmt()
			addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.UInt32, rc.next(x.typs.UInt32)))
			if !x.f.Config.BigEndian {
				// Int64Make args are big, little
				args[0], args[1] = args[1], args[0]
			}
			a = makeOf(a, ssaop.OpInt64Make, args)
			x.commonSelectors[sk] = a
			return a
		}
	}

	// An atomic type, either record the register or store it and update the memory.

	// Depending on the container Op, the leaves are either OpSelectN or OpArg{Int,Float}Reg

	if container.Op == ssaop.OpArg {
		if rc.hasRegs() {
			op, i := rc.ArgOpAndRegisterFor()
			name := container.Aux.(*ir.Name)
			a = makeOf(a, op, nil)
			a.AuxInt = i
			a.Aux = &ssa.AuxNameOffset{Name: name, Offset: rc.storeOffset}
		} else {
			key := selKey{container, rc.storeOffset, at.Size(), at}
			w := x.commonArgs[key]
			if w != nil && w.Uses != 0 {
				if a == nil {
					a = w
				} else {
					a.CopyOf(w)
				}
			} else {
				if a == nil {
					aux := container.Aux
					auxInt := container.AuxInt + rc.storeOffset
					a = container.Block.NewValue0IA(container.Pos, ssaop.OpArg, at, auxInt, aux)
				} else {
					// do nothing, the original should be okay.
				}
				x.commonArgs[key] = a
			}
		}
	} else if container.Op == ssaop.OpSelectN {
		call := container.Args[0]
		aux := call.Aux.(*ssa.AuxCall)
		which := container.AuxInt

		if at == types.TypeMem {
			if a != m0 || a != x.memForCall[call.ID] {
				panic(fmt.Errorf("Memories %s, %s, and %s should all be equal after %s", a.LongString(), m0.LongString(), x.memForCall[call.ID], call.LongString()))
			}
		} else if rc.hasRegs() {
			firstReg := uint32(0)
			for i := 0; i < int(which); i++ {
				firstReg += uint32(len(aux.AbiInfo.OutParam(i).Registers))
			}
			reg := int64(rc.nextSlice + Abi1RO(firstReg))
			a = makeOf(a, ssaop.OpSelectN, []*ssa.Value{call})
			a.AuxInt = reg
		} else {
			off := x.offsetFrom(x.f.Entry, x.sp, rc.storeOffset+aux.OffsetOfResult(which), types.NewPtr(at))
			a = makeOf(a, ssaop.OpLoad, []*ssa.Value{off, m0})
		}

	} else {
		panic(fmt.Errorf("Expected container OpArg or OpSelectN, saw %v instead", container.LongString()))
	}

	x.commonSelectors[sk] = a
	return a
}

// rewriteWideSelectToStores handles the case of a SelectN'd result from a function call that is too large for SSA,
// but is transferred in registers.  In this case the register cursor tracks both operands; the register sources and
// the memory destinations.
// This returns the memory flowing out of the last store
func (x *expandState) rewriteWideSelectToStores(pos src.XPos, b *ssa.Block, container, m0 *ssa.Value, at *types.Type, rc registerCursor) *ssa.Value {

	if at.Size() == 0 {
		return m0
	}

	switch at.Kind() {
	case types.TARRAY:
		et := at.Elem()
		for i := int64(0); i < at.NumElem(); i++ {
			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, et, rc.next(et))
		}
		return m0

	case types.TSTRUCT:
		// Assume ssagen/ssa.go (in buildssa) spills large aggregates so they won't appear here.
		if at.IsSIMD() {
			break // XXX
		}
		for i := 0; i < at.NumFields(); i++ {
			et := at.Field(i).Type
			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, et, rc.next(et))
			pos = pos.WithNotStmt()
		}
		return m0

	case types.TSLICE:
		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, at.Elem().PtrTo(), rc.next(x.typs.BytePtr))
		pos = pos.WithNotStmt()
		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Int, rc.next(x.typs.Int))
		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Int, rc.next(x.typs.Int))
		return m0

	case types.TSTRING:
		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.BytePtr, rc.next(x.typs.BytePtr))
		pos = pos.WithNotStmt()
		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Int, rc.next(x.typs.Int))
		return m0

	case types.TINTER:
		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Uintptr, rc.next(x.typs.Uintptr))
		pos = pos.WithNotStmt()
		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.BytePtr, rc.next(x.typs.BytePtr))
		return m0

	case types.TCOMPLEX64:
		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Float32, rc.next(x.typs.Float32))
		pos = pos.WithNotStmt()
		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Float32, rc.next(x.typs.Float32))
		return m0

	case types.TCOMPLEX128:
		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Float64, rc.next(x.typs.Float64))
		pos = pos.WithNotStmt()
		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Float64, rc.next(x.typs.Float64))
		return m0

	case types.TINT64:
		if at.Size() > x.regSize {
			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.firstType, rc.next(x.firstType))
			pos = pos.WithNotStmt()
			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.secondType, rc.next(x.secondType))
			return m0
		}
	case types.TUINT64:
		if at.Size() > x.regSize {
			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.UInt32, rc.next(x.typs.UInt32))
			pos = pos.WithNotStmt()
			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.UInt32, rc.next(x.typs.UInt32))
			return m0
		}
	}

	// TODO could change treatment of too-large OpArg, would deal with it here.
	if container.Op == ssaop.OpSelectN {
		call := container.Args[0]
		aux := call.Aux.(*ssa.AuxCall)
		which := container.AuxInt

		if rc.hasRegs() {
			firstReg := uint32(0)
			for i := 0; i < int(which); i++ {
				firstReg += uint32(len(aux.AbiInfo.OutParam(i).Registers))
			}
			reg := int64(rc.nextSlice + Abi1RO(firstReg))
			a := b.NewValue1I(pos, ssaop.OpSelectN, at, reg, call)
			dst := x.offsetFrom(b, rc.storeDest, rc.storeOffset, types.NewPtr(at))
			m0 = b.NewValue3A(pos, ssaop.OpStore, types.TypeMem, at, dst, a, m0)
		} else {
			panic(fmt.Errorf("Expected rc to have registers"))
		}
	} else {
		panic(fmt.Errorf("Expected container OpSelectN, saw %v instead", container.LongString()))
	}
	return m0
}

func isBlockMultiValueExit(b *ssa.Block) bool {
	return (b.Kind == block.BlockRet || b.Kind == block.BlockRetJmp) && b.Controls[0] != nil && b.Controls[0].Op == ssaop.OpMakeResult
}

type Abi1RO uint8 // An offset within a parameter's slice of register indices, for abi1.

// A registerCursor tracks which register is used for an Arg or regValues, or a piece of such.
type registerCursor struct {
	storeDest   *ssa.Value // if there are no register targets, then this is the base of the store.
	storeOffset int64
	regs        []abi.RegIndex // the registers available for this Arg/result (which is all in registers or not at all)
	nextSlice   Abi1RO         // the next register/register-slice offset
	config      *abi.ABIConfig
	regValues   *[]*ssa.Value // values assigned to registers accumulate here
}

func (c *registerCursor) String() string {
	dest := "<none>"
	if c.storeDest != nil {
		dest = fmt.Sprintf("%s+%d", c.storeDest.String(), c.storeOffset)
	}
	regs := "<none>"
	if c.regValues != nil {
		regs = ""
		for i, x := range *c.regValues {
			if i > 0 {
				regs = regs + "; "
			}
			regs = regs + x.LongString()
		}
	}

	// not printing the config because that has not been useful
	return fmt.Sprintf("RCSR{storeDest=%v, regsLen=%d, nextSlice=%d, regValues=[%s]}", dest, len(c.regs), c.nextSlice, regs)
}

// next effectively post-increments the register cursor; the receiver is advanced,
// the (aligned) old value is returned.
func (c *registerCursor) next(t *types.Type) registerCursor {
	c.storeOffset = types.RoundUp(c.storeOffset, t.Alignment())
	rc := *c
	c.storeOffset = types.RoundUp(c.storeOffset+t.Size(), t.Alignment())
	if int(c.nextSlice) < len(c.regs) {
		w := c.config.NumParamRegs(t)
		c.nextSlice += Abi1RO(w)
	}
	return rc
}

// plus returns a register cursor offset from the original, without modifying the original.
func (c *registerCursor) plus(regWidth Abi1RO) registerCursor {
	rc := *c
	rc.nextSlice += regWidth
	return rc
}

func (c *registerCursor) init(regs []abi.RegIndex, info *abi.ABIParamResultInfo, result *[]*ssa.Value, storeDest *ssa.Value, storeOffset int64) {
	c.regs = regs
	c.nextSlice = 0
	c.storeOffset = storeOffset
	c.storeDest = storeDest
	c.config = info.Config()
	c.regValues = result
}

func (c *registerCursor) addArg(v *ssa.Value) {
	*c.regValues = append(*c.regValues, v)
}

func (c *registerCursor) hasRegs() bool {
	return len(c.regs) > 0
}

func (c *registerCursor) ArgOpAndRegisterFor() (ssaop.Op, int64) {
	r := c.regs[c.nextSlice]
	return ssa.ArgOpAndRegisterFor(r, c.config)
}

type selKey struct {
	from          *ssa.Value // what is selected from
	offsetOrIndex int64      // whatever is appropriate for the selector
	size          int64
	typ           *types.Type
}

type expandState struct {
	f       *ssa.Func
	debug   int // odd values log lost statement markers, so likely settings are 1 (stmts), 2 (expansion), and 3 (both)
	regSize int64
	sp      *ssa.Value
	typs    *ssa.Types

	firstOp    ssaop.Op    // for 64-bit integers on 32-bit machines, first word in memory
	secondOp   ssaop.Op    // for 64-bit integers on 32-bit machines, second word in memory
	firstType  *types.Type // first half type, for Int64
	secondType *types.Type // second half type, for Int64

	wideSelects     map[*ssa.Value]*ssa.Value // Selects that are not SSA-able, mapped to consuming stores.
	commonSelectors map[selKey]*ssa.Value     // used to de-dupe selectors
	commonArgs      map[selKey]*ssa.Value     // used to de-dupe OpArg/OpArgIntReg/OpArgFloatReg
	memForCall      map[ssa.ID]*ssa.Value     // For a call, need to know the unique selector that gets the mem.
	indentLevel     int                       // Indentation for debugging recursion
}

// offsetFrom creates an offset from a pointer, simplifying chained offsets and offsets from SP
func (x *expandState) offsetFrom(b *ssa.Block, from *ssa.Value, offset int64, pt *types.Type) *ssa.Value {
	ft := from.Type
	if offset == 0 {
		if ft == pt {
			return from
		}
		// This captures common, (apparently) safe cases.  The unsafe cases involve ft == uintptr
		if (ft.IsPtr() || ft.IsUnsafePtr()) && pt.IsPtr() {
			return from
		}
	}
	// Simplify, canonicalize
	for from.Op == ssaop.OpOffPtr {
		offset += from.AuxInt
		from = from.Args[0]
	}
	if from == x.sp {
		return x.f.ConstOffPtrSP(pt, offset, x.sp)
	}
	return b.NewValue1I(from.Pos.WithNotStmt(), ssaop.OpOffPtr, pt, offset, from)
}

// prAssignForArg returns the ABIParamAssignment for v, assumed to be an OpArg.
func (x *expandState) prAssignForArg(v *ssa.Value) *abi.ABIParamAssignment {
	if v.Op != ssaop.OpArg {
		panic(fmt.Errorf("Wanted OpArg, instead saw %s", v.LongString()))
	}
	return ssa.ParamAssignmentForArgName(x.f, v.Aux.(*ir.Name))
}

// indent increments (or decrements) the indentation.
func (x *expandState) indent(n int) {
	x.indentLevel += n
}

// Printf does an indented fmt.Printf on the format and args.
func (x *expandState) Printf(format string, a ...any) (n int, err error) {
	if x.indentLevel > 0 {
		fmt.Printf("%[1]*s", x.indentLevel, "")
	}
	return fmt.Printf(format, a...)
}

func (x *expandState) invalidateRecursively(a *ssa.Value) {
	var s string
	if x.debug > 0 {
		plus := " "
		if a.Pos.IsStmt() == src.PosIsStmt {
			plus = " +"
		}
		s = a.String() + plus + a.Pos.LineNumber() + " " + a.LongString()
		if x.debug > 1 {
			x.Printf("...marking %v unused\n", s)
		}
	}
	lost := a.InvalidateRecursively()
	if x.debug&1 != 0 && lost { // For odd values of x.debug, do this.
		x.Printf("Lost statement marker in %s on former %s\n", base.Ctxt.Pkgpath+"."+x.f.Name, s)
	}
}