src/cmd/compile/internal/ssa/looprotate.go

// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ssa

import (
	"slices"
)

// loopRotate converts loops with a check-loop-condition-at-beginning
// to loops with a check-loop-condition-at-end.
// This helps loops avoid extra unnecessary jumps.
//
//	 loop:
//	   CMPQ ...
//	   JGE exit
//	   ...
//	   JMP loop
//	 exit:
//
//	  JMP entry
//	loop:
//	  ...
//	entry:
//	  CMPQ ...
//	  JLT loop
func loopRotate(f *Func) {
	loopnest := f.loopnest()
	if loopnest.hasIrreducible {
		return
	}
	if len(loopnest.loops) == 0 {
		return
	}

	idToIdx := f.Cache.allocIntSlice(f.NumBlocks())
	defer f.Cache.freeIntSlice(idToIdx)
	for i, b := range f.Blocks {
		idToIdx[b.ID] = i
	}

	// Set of blocks we're moving, by ID.
	move := map[ID]struct{}{}

	// Map from block ID to the moving blocks that should
	// come right after it.
	// If a block, which has its ID present in keys of the 'after' map,
	// occurs in some other block's 'after' list, that represents whole
	// nested loop, e.g. consider an inner loop I nested into an outer
	// loop O. It and Ot are corresponding top block for these loops
	// chosen by our algorithm, and It is in the Ot's 'after' list.
	//
	//    Before:                     After:
	//
	//       e                       e
	//       │                       │
	//       │                       │Ot ◄───┐
	//       ▼                       ▼▼      │
	//   ┌───Oh ◄────┐           ┌─┬─Oh      │
	//   │   │       │           │ │         │
	//   │   │       │           │ │ It◄───┐ │
	//   │   ▼       │           │ │ ▼     │ │
	//   │ ┌─Ih◄───┐ │           │ └►Ih    │ │
	//   │ │ │     │ │           │ ┌─┤     │ │
	//   │ │ ▼     │ │           │ │ ▼     │ │
	//   │ │ Ib    │ │           │ │ Ib    │ │
	//   │ │ └─►It─┘ │           │ │ └─────┘ │
	//   │ │         │           │ │         │
	//   │ └►Ie      │           │ └►Ie      │
	//   │   └─►Ot───┘           │   └───────┘
	//   │                       │
	//   └──►Oe                  └──►Oe
	//
	// We build the 'after' lists for each of the top blocks Ot and It:
	//   after[Ot]: Oh, It, Ie
	//   after[It]: Ih, Ib
	after := map[ID][]*Block{}

	// Map from loop header ID to the new top block for the loop.
	tops := map[ID]*Block{}

	// Order loops to rotate any child loop before adding its top block
	// to the parent loop's 'after' list.
	loopOrder := f.Cache.allocIntSlice(len(loopnest.loops))
	for i := range loopOrder {
		loopOrder[i] = i
	}
	defer f.Cache.freeIntSlice(loopOrder)
	slices.SortFunc(loopOrder, func(i, j int) int {
		di := loopnest.loops[i].depth
		dj := loopnest.loops[j].depth
		switch {
		case di > dj:
			return -1
		case di < dj:
			return 1
		default:
			return 0
		}
	})

	// Check each loop header and decide if we want to move it.
	for _, loopIdx := range loopOrder {
		loop := loopnest.loops[loopIdx]
		b := loop.header
		var p *Block // b's in-loop predecessor
		for _, e := range b.Preds {
			if e.b.Kind != BlockPlain {
				continue
			}
			if loopnest.b2l[e.b.ID] != loop {
				continue
			}
			p = e.b
		}
		if p == nil {
			continue
		}
		tops[loop.header.ID] = p
		p.Hotness |= HotInitial
		if f.IsPgoHot {
			p.Hotness |= HotPgo
		}
		// blocks will be arranged so that p is ordered first, if it isn't already.
		if p == b { // p is header, already first (and also, only block in the loop)
			continue
		}
		p.Hotness |= HotNotFlowIn

		// the loop header b follows p
		after[p.ID] = []*Block{b}
		for {
			nextIdx := idToIdx[b.ID] + 1
			if nextIdx >= len(f.Blocks) { // reached end of function (maybe impossible?)
				break
			}
			nextb := f.Blocks[nextIdx]
			if nextb == p { // original loop predecessor is next
				break
			}
			if bloop := loopnest.b2l[nextb.ID]; bloop != nil {
				if bloop == loop || bloop.outer == loop && tops[bloop.header.ID] == nextb {
					after[p.ID] = append(after[p.ID], nextb)
				}
			}
			b = nextb
		}
		// Swap b and p so that we'll handle p before b when moving blocks.
		f.Blocks[idToIdx[loop.header.ID]] = p
		f.Blocks[idToIdx[p.ID]] = loop.header
		idToIdx[loop.header.ID], idToIdx[p.ID] = idToIdx[p.ID], idToIdx[loop.header.ID]

		// Place loop blocks after p.
		for _, b := range after[p.ID] {
			move[b.ID] = struct{}{}
		}
	}

	// Move blocks to their destinations in a single pass.
	// We rely here on the fact that loop headers must come
	// before the rest of the loop.  And that relies on the
	// fact that we only identify reducible loops.
	j := 0
	// Some blocks that are not part of a loop may be placed
	// between loop blocks. In order to avoid these blocks from
	// being overwritten, use a temporary slice.
	oldOrder := f.Cache.allocBlockSlice(len(f.Blocks))
	defer f.Cache.freeBlockSlice(oldOrder)
	copy(oldOrder, f.Blocks)
	var moveBlocks func(bs []*Block)
	moveBlocks = func(blocks []*Block) {
		for _, a := range blocks {
			f.Blocks[j] = a
			j++
			if nextBlocks, ok := after[a.ID]; ok {
				moveBlocks(nextBlocks)
			}
		}
	}
	for _, b := range oldOrder {
		if _, ok := move[b.ID]; ok {
			continue
		}
		f.Blocks[j] = b
		j++
		moveBlocks(after[b.ID])
	}
	if j != len(oldOrder) {
		f.Fatalf("bad reordering in looprotate")
	}
}