blob: dd1f39dbef74398f7cf9e338d26d6f9b1a8f4ca7 [file] [log] [blame]
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package ssa
import (
"cmd/compile/internal/base"
"cmd/compile/internal/types"
"fmt"
)
type indVarFlags uint8
const (
indVarMinExc indVarFlags = 1 << iota // minimum value is exclusive (default: inclusive)
indVarMaxInc // maximum value is inclusive (default: exclusive)
indVarCountDown // if set the iteration starts at max and count towards min (default: min towards max)
)
type indVar struct {
ind *Value // induction variable
nxt *Value // the incremented variable
min *Value // minimum value, inclusive/exclusive depends on flags
max *Value // maximum value, inclusive/exclusive depends on flags
entry *Block // entry block in the loop.
flags indVarFlags
// Invariant: for all blocks strictly dominated by entry:
// min <= ind < max [if flags == 0]
// min < ind < max [if flags == indVarMinExc]
// min <= ind <= max [if flags == indVarMaxInc]
// min < ind <= max [if flags == indVarMinExc|indVarMaxInc]
}
// parseIndVar checks whether the SSA value passed as argument is a valid induction
// variable, and, if so, extracts:
// - the minimum bound
// - the increment value
// - the "next" value (SSA value that is Phi'd into the induction variable every loop)
//
// Currently, we detect induction variables that match (Phi min nxt),
// with nxt being (Add inc ind).
// If it can't parse the induction variable correctly, it returns (nil, nil, nil).
func parseIndVar(ind *Value) (min, inc, nxt *Value) {
if ind.Op != OpPhi {
return
}
if n := ind.Args[0]; (n.Op == OpAdd64 || n.Op == OpAdd32 || n.Op == OpAdd16 || n.Op == OpAdd8) && (n.Args[0] == ind || n.Args[1] == ind) {
min, nxt = ind.Args[1], n
} else if n := ind.Args[1]; (n.Op == OpAdd64 || n.Op == OpAdd32 || n.Op == OpAdd16 || n.Op == OpAdd8) && (n.Args[0] == ind || n.Args[1] == ind) {
min, nxt = ind.Args[0], n
} else {
// Not a recognized induction variable.
return
}
if nxt.Args[0] == ind { // nxt = ind + inc
inc = nxt.Args[1]
} else if nxt.Args[1] == ind { // nxt = inc + ind
inc = nxt.Args[0]
} else {
panic("unreachable") // one of the cases must be true from the above.
}
return
}
// findIndVar finds induction variables in a function.
//
// Look for variables and blocks that satisfy the following
//
// loop:
// ind = (Phi min nxt),
// if ind < max
// then goto enter_loop
// else goto exit_loop
//
// enter_loop:
// do something
// nxt = inc + ind
// goto loop
//
// exit_loop:
func findIndVar(f *Func) []indVar {
var iv []indVar
sdom := f.Sdom()
for _, b := range f.Blocks {
if b.Kind != BlockIf || len(b.Preds) != 2 {
continue
}
var ind *Value // induction variable
var init *Value // starting value
var limit *Value // ending value
// Check that the control if it either ind </<= limit or limit </<= ind.
// TODO: Handle unsigned comparisons?
c := b.Controls[0]
inclusive := false
switch c.Op {
case OpLeq64, OpLeq32, OpLeq16, OpLeq8:
inclusive = true
fallthrough
case OpLess64, OpLess32, OpLess16, OpLess8:
ind, limit = c.Args[0], c.Args[1]
default:
continue
}
// See if this is really an induction variable
less := true
init, inc, nxt := parseIndVar(ind)
if init == nil {
// We failed to parse the induction variable. Before punting, we want to check
// whether the control op was written with the induction variable on the RHS
// instead of the LHS. This happens for the downwards case, like:
// for i := len(n)-1; i >= 0; i--
init, inc, nxt = parseIndVar(limit)
if init == nil {
// No recognized induction variable on either operand
continue
}
// Ok, the arguments were reversed. Swap them, and remember that we're
// looking at an ind >/>= loop (so the induction must be decrementing).
ind, limit = limit, ind
less = false
}
if ind.Block != b {
// TODO: Could be extended to include disjointed loop headers.
// I don't think this is causing missed optimizations in real world code often.
// See https://go.dev/issue/63955
continue
}
// Expect the increment to be a nonzero constant.
if !inc.isGenericIntConst() {
continue
}
step := inc.AuxInt
if step == 0 {
continue
}
// Increment sign must match comparison direction.
// When incrementing, the termination comparison must be ind </<= limit.
// When decrementing, the termination comparison must be ind >/>= limit.
// See issue 26116.
if step > 0 && !less {
continue
}
if step < 0 && less {
continue
}
// Up to now we extracted the induction variable (ind),
// the increment delta (inc), the temporary sum (nxt),
// the initial value (init) and the limiting value (limit).
//
// We also know that ind has the form (Phi init nxt) where
// nxt is (Add inc nxt) which means: 1) inc dominates nxt
// and 2) there is a loop starting at inc and containing nxt.
//
// We need to prove that the induction variable is incremented
// only when it's smaller than the limiting value.
// Two conditions must happen listed below to accept ind
// as an induction variable.
// First condition: loop entry has a single predecessor, which
// is the header block. This implies that b.Succs[0] is
// reached iff ind < limit.
if len(b.Succs[0].b.Preds) != 1 {
// b.Succs[1] must exit the loop.
continue
}
// Second condition: b.Succs[0] dominates nxt so that
// nxt is computed when inc < limit.
if !sdom.IsAncestorEq(b.Succs[0].b, nxt.Block) {
// inc+ind can only be reached through the branch that enters the loop.
continue
}
// Check for overflow/underflow. We need to make sure that inc never causes
// the induction variable to wrap around.
// We use a function wrapper here for easy return true / return false / keep going logic.
// This function returns true if the increment will never overflow/underflow.
ok := func() bool {
if step > 0 {
if limit.isGenericIntConst() {
// Figure out the actual largest value.
v := limit.AuxInt
if !inclusive {
if v == minSignedValue(limit.Type) {
return false // < minint is never satisfiable.
}
v--
}
if init.isGenericIntConst() {
// Use stride to compute a better lower limit.
if init.AuxInt > v {
return false
}
v = addU(init.AuxInt, diff(v, init.AuxInt)/uint64(step)*uint64(step))
}
if addWillOverflow(v, step) {
return false
}
if inclusive && v != limit.AuxInt || !inclusive && v+1 != limit.AuxInt {
// We know a better limit than the programmer did. Use our limit instead.
limit = f.constVal(limit.Op, limit.Type, v, true)
inclusive = true
}
return true
}
if step == 1 && !inclusive {
// Can't overflow because maxint is never a possible value.
return true
}
// If the limit is not a constant, check to see if it is a
// negative offset from a known non-negative value.
knn, k := findKNN(limit)
if knn == nil || k < 0 {
return false
}
// limit == (something nonnegative) - k. That subtraction can't underflow, so
// we can trust it.
if inclusive {
// ind <= knn - k cannot overflow if step is at most k
return step <= k
}
// ind < knn - k cannot overflow if step is at most k+1
return step <= k+1 && k != maxSignedValue(limit.Type)
} else { // step < 0
if limit.Op == OpConst64 {
// Figure out the actual smallest value.
v := limit.AuxInt
if !inclusive {
if v == maxSignedValue(limit.Type) {
return false // > maxint is never satisfiable.
}
v++
}
if init.isGenericIntConst() {
// Use stride to compute a better lower limit.
if init.AuxInt < v {
return false
}
v = subU(init.AuxInt, diff(init.AuxInt, v)/uint64(-step)*uint64(-step))
}
if subWillUnderflow(v, -step) {
return false
}
if inclusive && v != limit.AuxInt || !inclusive && v-1 != limit.AuxInt {
// We know a better limit than the programmer did. Use our limit instead.
limit = f.constVal(limit.Op, limit.Type, v, true)
inclusive = true
}
return true
}
if step == -1 && !inclusive {
// Can't underflow because minint is never a possible value.
return true
}
}
return false
}
if ok() {
flags := indVarFlags(0)
var min, max *Value
if step > 0 {
min = init
max = limit
if inclusive {
flags |= indVarMaxInc
}
} else {
min = limit
max = init
flags |= indVarMaxInc
if !inclusive {
flags |= indVarMinExc
}
flags |= indVarCountDown
step = -step
}
if f.pass.debug >= 1 {
printIndVar(b, ind, min, max, step, flags)
}
iv = append(iv, indVar{
ind: ind,
nxt: nxt,
min: min,
max: max,
entry: b.Succs[0].b,
flags: flags,
})
b.Logf("found induction variable %v (inc = %v, min = %v, max = %v)\n", ind, inc, min, max)
}
// TODO: other unrolling idioms
// for i := 0; i < KNN - KNN % k ; i += k
// for i := 0; i < KNN&^(k-1) ; i += k // k a power of 2
// for i := 0; i < KNN&(-k) ; i += k // k a power of 2
}
return iv
}
// addWillOverflow reports whether x+y would result in a value more than maxint.
func addWillOverflow(x, y int64) bool {
return x+y < x
}
// subWillUnderflow reports whether x-y would result in a value less than minint.
func subWillUnderflow(x, y int64) bool {
return x-y > x
}
// diff returns x-y as a uint64. Requires x>=y.
func diff(x, y int64) uint64 {
if x < y {
base.Fatalf("diff %d - %d underflowed", x, y)
}
return uint64(x - y)
}
// addU returns x+y. Requires that x+y does not overflow an int64.
func addU(x int64, y uint64) int64 {
if y >= 1<<63 {
if x >= 0 {
base.Fatalf("addU overflowed %d + %d", x, y)
}
x += 1<<63 - 1
x += 1
y -= 1 << 63
}
if addWillOverflow(x, int64(y)) {
base.Fatalf("addU overflowed %d + %d", x, y)
}
return x + int64(y)
}
// subU returns x-y. Requires that x-y does not underflow an int64.
func subU(x int64, y uint64) int64 {
if y >= 1<<63 {
if x < 0 {
base.Fatalf("subU underflowed %d - %d", x, y)
}
x -= 1<<63 - 1
x -= 1
y -= 1 << 63
}
if subWillUnderflow(x, int64(y)) {
base.Fatalf("subU underflowed %d - %d", x, y)
}
return x - int64(y)
}
// if v is known to be x - c, where x is known to be nonnegative and c is a
// constant, return x, c. Otherwise return nil, 0.
func findKNN(v *Value) (*Value, int64) {
var x, y *Value
x = v
switch v.Op {
case OpSub64, OpSub32, OpSub16, OpSub8:
x = v.Args[0]
y = v.Args[1]
case OpAdd64, OpAdd32, OpAdd16, OpAdd8:
x = v.Args[0]
y = v.Args[1]
if x.isGenericIntConst() {
x, y = y, x
}
}
switch x.Op {
case OpSliceLen, OpStringLen, OpSliceCap:
default:
return nil, 0
}
if y == nil {
return x, 0
}
if !y.isGenericIntConst() {
return nil, 0
}
if v.Op == OpAdd64 || v.Op == OpAdd32 || v.Op == OpAdd16 || v.Op == OpAdd8 {
return x, -y.AuxInt
}
return x, y.AuxInt
}
func printIndVar(b *Block, i, min, max *Value, inc int64, flags indVarFlags) {
mb1, mb2 := "[", "]"
if flags&indVarMinExc != 0 {
mb1 = "("
}
if flags&indVarMaxInc == 0 {
mb2 = ")"
}
mlim1, mlim2 := fmt.Sprint(min.AuxInt), fmt.Sprint(max.AuxInt)
if !min.isGenericIntConst() {
if b.Func.pass.debug >= 2 {
mlim1 = fmt.Sprint(min)
} else {
mlim1 = "?"
}
}
if !max.isGenericIntConst() {
if b.Func.pass.debug >= 2 {
mlim2 = fmt.Sprint(max)
} else {
mlim2 = "?"
}
}
extra := ""
if b.Func.pass.debug >= 2 {
extra = fmt.Sprintf(" (%s)", i)
}
b.Func.Warnl(b.Pos, "Induction variable: limits %v%v,%v%v, increment %d%s", mb1, mlim1, mlim2, mb2, inc, extra)
}
func minSignedValue(t *types.Type) int64 {
return -1 << (t.Size()*8 - 1)
}
func maxSignedValue(t *types.Type) int64 {
return 1<<((t.Size()*8)-1) - 1
}