| // Copyright 2009 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| package walk |
| |
| import ( |
| "encoding/binary" |
| "go/constant" |
| |
| "cmd/compile/internal/base" |
| "cmd/compile/internal/ir" |
| "cmd/compile/internal/reflectdata" |
| "cmd/compile/internal/ssagen" |
| "cmd/compile/internal/typecheck" |
| "cmd/compile/internal/types" |
| "cmd/internal/sys" |
| ) |
| |
| // The result of walkCompare MUST be assigned back to n, e.g. |
| // n.Left = walkCompare(n.Left, init) |
| func walkCompare(n *ir.BinaryExpr, init *ir.Nodes) ir.Node { |
| if n.X.Type().IsInterface() && n.Y.Type().IsInterface() && n.X.Op() != ir.ONIL && n.Y.Op() != ir.ONIL { |
| return walkCompareInterface(n, init) |
| } |
| |
| if n.X.Type().IsString() && n.Y.Type().IsString() { |
| return walkCompareString(n, init) |
| } |
| |
| n.X = walkExpr(n.X, init) |
| n.Y = walkExpr(n.Y, init) |
| |
| // Given mixed interface/concrete comparison, |
| // rewrite into types-equal && data-equal. |
| // This is efficient, avoids allocations, and avoids runtime calls. |
| if n.X.Type().IsInterface() != n.Y.Type().IsInterface() { |
| // Preserve side-effects in case of short-circuiting; see #32187. |
| l := cheapExpr(n.X, init) |
| r := cheapExpr(n.Y, init) |
| // Swap so that l is the interface value and r is the concrete value. |
| if n.Y.Type().IsInterface() { |
| l, r = r, l |
| } |
| |
| // Handle both == and !=. |
| eq := n.Op() |
| andor := ir.OOROR |
| if eq == ir.OEQ { |
| andor = ir.OANDAND |
| } |
| // Check for types equal. |
| // For empty interface, this is: |
| // l.tab == type(r) |
| // For non-empty interface, this is: |
| // l.tab != nil && l.tab._type == type(r) |
| var eqtype ir.Node |
| tab := ir.NewUnaryExpr(base.Pos, ir.OITAB, l) |
| rtyp := reflectdata.TypePtr(r.Type()) |
| if l.Type().IsEmptyInterface() { |
| tab.SetType(types.NewPtr(types.Types[types.TUINT8])) |
| tab.SetTypecheck(1) |
| eqtype = ir.NewBinaryExpr(base.Pos, eq, tab, rtyp) |
| } else { |
| nonnil := ir.NewBinaryExpr(base.Pos, brcom(eq), typecheck.NodNil(), tab) |
| match := ir.NewBinaryExpr(base.Pos, eq, itabType(tab), rtyp) |
| eqtype = ir.NewLogicalExpr(base.Pos, andor, nonnil, match) |
| } |
| // Check for data equal. |
| eqdata := ir.NewBinaryExpr(base.Pos, eq, ifaceData(n.Pos(), l, r.Type()), r) |
| // Put it all together. |
| expr := ir.NewLogicalExpr(base.Pos, andor, eqtype, eqdata) |
| return finishCompare(n, expr, init) |
| } |
| |
| // Must be comparison of array or struct. |
| // Otherwise back end handles it. |
| // While we're here, decide whether to |
| // inline or call an eq alg. |
| t := n.X.Type() |
| var inline bool |
| |
| maxcmpsize := int64(4) |
| unalignedLoad := canMergeLoads() |
| if unalignedLoad { |
| // Keep this low enough to generate less code than a function call. |
| maxcmpsize = 2 * int64(ssagen.Arch.LinkArch.RegSize) |
| } |
| |
| switch t.Kind() { |
| default: |
| if base.Debug.Libfuzzer != 0 && t.IsInteger() { |
| n.X = cheapExpr(n.X, init) |
| n.Y = cheapExpr(n.Y, init) |
| |
| // If exactly one comparison operand is |
| // constant, invoke the constcmp functions |
| // instead, and arrange for the constant |
| // operand to be the first argument. |
| l, r := n.X, n.Y |
| if r.Op() == ir.OLITERAL { |
| l, r = r, l |
| } |
| constcmp := l.Op() == ir.OLITERAL && r.Op() != ir.OLITERAL |
| |
| var fn string |
| var paramType *types.Type |
| switch t.Size() { |
| case 1: |
| fn = "libfuzzerTraceCmp1" |
| if constcmp { |
| fn = "libfuzzerTraceConstCmp1" |
| } |
| paramType = types.Types[types.TUINT8] |
| case 2: |
| fn = "libfuzzerTraceCmp2" |
| if constcmp { |
| fn = "libfuzzerTraceConstCmp2" |
| } |
| paramType = types.Types[types.TUINT16] |
| case 4: |
| fn = "libfuzzerTraceCmp4" |
| if constcmp { |
| fn = "libfuzzerTraceConstCmp4" |
| } |
| paramType = types.Types[types.TUINT32] |
| case 8: |
| fn = "libfuzzerTraceCmp8" |
| if constcmp { |
| fn = "libfuzzerTraceConstCmp8" |
| } |
| paramType = types.Types[types.TUINT64] |
| default: |
| base.Fatalf("unexpected integer size %d for %v", t.Size(), t) |
| } |
| init.Append(mkcall(fn, nil, init, tracecmpArg(l, paramType, init), tracecmpArg(r, paramType, init))) |
| } |
| return n |
| case types.TARRAY: |
| // We can compare several elements at once with 2/4/8 byte integer compares |
| inline = t.NumElem() <= 1 || (types.IsSimple[t.Elem().Kind()] && (t.NumElem() <= 4 || t.Elem().Width*t.NumElem() <= maxcmpsize)) |
| case types.TSTRUCT: |
| inline = t.NumComponents(types.IgnoreBlankFields) <= 4 |
| } |
| |
| cmpl := n.X |
| for cmpl != nil && cmpl.Op() == ir.OCONVNOP { |
| cmpl = cmpl.(*ir.ConvExpr).X |
| } |
| cmpr := n.Y |
| for cmpr != nil && cmpr.Op() == ir.OCONVNOP { |
| cmpr = cmpr.(*ir.ConvExpr).X |
| } |
| |
| // Chose not to inline. Call equality function directly. |
| if !inline { |
| // eq algs take pointers; cmpl and cmpr must be addressable |
| if !ir.IsAddressable(cmpl) || !ir.IsAddressable(cmpr) { |
| base.Fatalf("arguments of comparison must be lvalues - %v %v", cmpl, cmpr) |
| } |
| |
| fn, needsize := eqFor(t) |
| call := ir.NewCallExpr(base.Pos, ir.OCALL, fn, nil) |
| call.Args.Append(typecheck.NodAddr(cmpl)) |
| call.Args.Append(typecheck.NodAddr(cmpr)) |
| if needsize { |
| call.Args.Append(ir.NewInt(t.Width)) |
| } |
| res := ir.Node(call) |
| if n.Op() != ir.OEQ { |
| res = ir.NewUnaryExpr(base.Pos, ir.ONOT, res) |
| } |
| return finishCompare(n, res, init) |
| } |
| |
| // inline: build boolean expression comparing element by element |
| andor := ir.OANDAND |
| if n.Op() == ir.ONE { |
| andor = ir.OOROR |
| } |
| var expr ir.Node |
| compare := func(el, er ir.Node) { |
| a := ir.NewBinaryExpr(base.Pos, n.Op(), el, er) |
| if expr == nil { |
| expr = a |
| } else { |
| expr = ir.NewLogicalExpr(base.Pos, andor, expr, a) |
| } |
| } |
| cmpl = safeExpr(cmpl, init) |
| cmpr = safeExpr(cmpr, init) |
| if t.IsStruct() { |
| for _, f := range t.Fields().Slice() { |
| sym := f.Sym |
| if sym.IsBlank() { |
| continue |
| } |
| compare( |
| ir.NewSelectorExpr(base.Pos, ir.OXDOT, cmpl, sym), |
| ir.NewSelectorExpr(base.Pos, ir.OXDOT, cmpr, sym), |
| ) |
| } |
| } else { |
| step := int64(1) |
| remains := t.NumElem() * t.Elem().Width |
| combine64bit := unalignedLoad && types.RegSize == 8 && t.Elem().Width <= 4 && t.Elem().IsInteger() |
| combine32bit := unalignedLoad && t.Elem().Width <= 2 && t.Elem().IsInteger() |
| combine16bit := unalignedLoad && t.Elem().Width == 1 && t.Elem().IsInteger() |
| for i := int64(0); remains > 0; { |
| var convType *types.Type |
| switch { |
| case remains >= 8 && combine64bit: |
| convType = types.Types[types.TINT64] |
| step = 8 / t.Elem().Width |
| case remains >= 4 && combine32bit: |
| convType = types.Types[types.TUINT32] |
| step = 4 / t.Elem().Width |
| case remains >= 2 && combine16bit: |
| convType = types.Types[types.TUINT16] |
| step = 2 / t.Elem().Width |
| default: |
| step = 1 |
| } |
| if step == 1 { |
| compare( |
| ir.NewIndexExpr(base.Pos, cmpl, ir.NewInt(i)), |
| ir.NewIndexExpr(base.Pos, cmpr, ir.NewInt(i)), |
| ) |
| i++ |
| remains -= t.Elem().Width |
| } else { |
| elemType := t.Elem().ToUnsigned() |
| cmplw := ir.Node(ir.NewIndexExpr(base.Pos, cmpl, ir.NewInt(i))) |
| cmplw = typecheck.Conv(cmplw, elemType) // convert to unsigned |
| cmplw = typecheck.Conv(cmplw, convType) // widen |
| cmprw := ir.Node(ir.NewIndexExpr(base.Pos, cmpr, ir.NewInt(i))) |
| cmprw = typecheck.Conv(cmprw, elemType) |
| cmprw = typecheck.Conv(cmprw, convType) |
| // For code like this: uint32(s[0]) | uint32(s[1])<<8 | uint32(s[2])<<16 ... |
| // ssa will generate a single large load. |
| for offset := int64(1); offset < step; offset++ { |
| lb := ir.Node(ir.NewIndexExpr(base.Pos, cmpl, ir.NewInt(i+offset))) |
| lb = typecheck.Conv(lb, elemType) |
| lb = typecheck.Conv(lb, convType) |
| lb = ir.NewBinaryExpr(base.Pos, ir.OLSH, lb, ir.NewInt(8*t.Elem().Width*offset)) |
| cmplw = ir.NewBinaryExpr(base.Pos, ir.OOR, cmplw, lb) |
| rb := ir.Node(ir.NewIndexExpr(base.Pos, cmpr, ir.NewInt(i+offset))) |
| rb = typecheck.Conv(rb, elemType) |
| rb = typecheck.Conv(rb, convType) |
| rb = ir.NewBinaryExpr(base.Pos, ir.OLSH, rb, ir.NewInt(8*t.Elem().Width*offset)) |
| cmprw = ir.NewBinaryExpr(base.Pos, ir.OOR, cmprw, rb) |
| } |
| compare(cmplw, cmprw) |
| i += step |
| remains -= step * t.Elem().Width |
| } |
| } |
| } |
| if expr == nil { |
| expr = ir.NewBool(n.Op() == ir.OEQ) |
| // We still need to use cmpl and cmpr, in case they contain |
| // an expression which might panic. See issue 23837. |
| t := typecheck.Temp(cmpl.Type()) |
| a1 := typecheck.Stmt(ir.NewAssignStmt(base.Pos, t, cmpl)) |
| a2 := typecheck.Stmt(ir.NewAssignStmt(base.Pos, t, cmpr)) |
| init.Append(a1, a2) |
| } |
| return finishCompare(n, expr, init) |
| } |
| |
| func walkCompareInterface(n *ir.BinaryExpr, init *ir.Nodes) ir.Node { |
| n.Y = cheapExpr(n.Y, init) |
| n.X = cheapExpr(n.X, init) |
| eqtab, eqdata := reflectdata.EqInterface(n.X, n.Y) |
| var cmp ir.Node |
| if n.Op() == ir.OEQ { |
| cmp = ir.NewLogicalExpr(base.Pos, ir.OANDAND, eqtab, eqdata) |
| } else { |
| eqtab.SetOp(ir.ONE) |
| cmp = ir.NewLogicalExpr(base.Pos, ir.OOROR, eqtab, ir.NewUnaryExpr(base.Pos, ir.ONOT, eqdata)) |
| } |
| return finishCompare(n, cmp, init) |
| } |
| |
| func walkCompareString(n *ir.BinaryExpr, init *ir.Nodes) ir.Node { |
| // Rewrite comparisons to short constant strings as length+byte-wise comparisons. |
| var cs, ncs ir.Node // const string, non-const string |
| switch { |
| case ir.IsConst(n.X, constant.String) && ir.IsConst(n.Y, constant.String): |
| // ignore; will be constant evaluated |
| case ir.IsConst(n.X, constant.String): |
| cs = n.X |
| ncs = n.Y |
| case ir.IsConst(n.Y, constant.String): |
| cs = n.Y |
| ncs = n.X |
| } |
| if cs != nil { |
| cmp := n.Op() |
| // Our comparison below assumes that the non-constant string |
| // is on the left hand side, so rewrite "" cmp x to x cmp "". |
| // See issue 24817. |
| if ir.IsConst(n.X, constant.String) { |
| cmp = brrev(cmp) |
| } |
| |
| // maxRewriteLen was chosen empirically. |
| // It is the value that minimizes cmd/go file size |
| // across most architectures. |
| // See the commit description for CL 26758 for details. |
| maxRewriteLen := 6 |
| // Some architectures can load unaligned byte sequence as 1 word. |
| // So we can cover longer strings with the same amount of code. |
| canCombineLoads := canMergeLoads() |
| combine64bit := false |
| if canCombineLoads { |
| // Keep this low enough to generate less code than a function call. |
| maxRewriteLen = 2 * ssagen.Arch.LinkArch.RegSize |
| combine64bit = ssagen.Arch.LinkArch.RegSize >= 8 |
| } |
| |
| var and ir.Op |
| switch cmp { |
| case ir.OEQ: |
| and = ir.OANDAND |
| case ir.ONE: |
| and = ir.OOROR |
| default: |
| // Don't do byte-wise comparisons for <, <=, etc. |
| // They're fairly complicated. |
| // Length-only checks are ok, though. |
| maxRewriteLen = 0 |
| } |
| if s := ir.StringVal(cs); len(s) <= maxRewriteLen { |
| if len(s) > 0 { |
| ncs = safeExpr(ncs, init) |
| } |
| r := ir.Node(ir.NewBinaryExpr(base.Pos, cmp, ir.NewUnaryExpr(base.Pos, ir.OLEN, ncs), ir.NewInt(int64(len(s))))) |
| remains := len(s) |
| for i := 0; remains > 0; { |
| if remains == 1 || !canCombineLoads { |
| cb := ir.NewInt(int64(s[i])) |
| ncb := ir.NewIndexExpr(base.Pos, ncs, ir.NewInt(int64(i))) |
| r = ir.NewLogicalExpr(base.Pos, and, r, ir.NewBinaryExpr(base.Pos, cmp, ncb, cb)) |
| remains-- |
| i++ |
| continue |
| } |
| var step int |
| var convType *types.Type |
| switch { |
| case remains >= 8 && combine64bit: |
| convType = types.Types[types.TINT64] |
| step = 8 |
| case remains >= 4: |
| convType = types.Types[types.TUINT32] |
| step = 4 |
| case remains >= 2: |
| convType = types.Types[types.TUINT16] |
| step = 2 |
| } |
| ncsubstr := typecheck.Conv(ir.NewIndexExpr(base.Pos, ncs, ir.NewInt(int64(i))), convType) |
| csubstr := int64(s[i]) |
| // Calculate large constant from bytes as sequence of shifts and ors. |
| // Like this: uint32(s[0]) | uint32(s[1])<<8 | uint32(s[2])<<16 ... |
| // ssa will combine this into a single large load. |
| for offset := 1; offset < step; offset++ { |
| b := typecheck.Conv(ir.NewIndexExpr(base.Pos, ncs, ir.NewInt(int64(i+offset))), convType) |
| b = ir.NewBinaryExpr(base.Pos, ir.OLSH, b, ir.NewInt(int64(8*offset))) |
| ncsubstr = ir.NewBinaryExpr(base.Pos, ir.OOR, ncsubstr, b) |
| csubstr |= int64(s[i+offset]) << uint8(8*offset) |
| } |
| csubstrPart := ir.NewInt(csubstr) |
| // Compare "step" bytes as once |
| r = ir.NewLogicalExpr(base.Pos, and, r, ir.NewBinaryExpr(base.Pos, cmp, csubstrPart, ncsubstr)) |
| remains -= step |
| i += step |
| } |
| return finishCompare(n, r, init) |
| } |
| } |
| |
| var r ir.Node |
| if n.Op() == ir.OEQ || n.Op() == ir.ONE { |
| // prepare for rewrite below |
| n.X = cheapExpr(n.X, init) |
| n.Y = cheapExpr(n.Y, init) |
| eqlen, eqmem := reflectdata.EqString(n.X, n.Y) |
| // quick check of len before full compare for == or !=. |
| // memequal then tests equality up to length len. |
| if n.Op() == ir.OEQ { |
| // len(left) == len(right) && memequal(left, right, len) |
| r = ir.NewLogicalExpr(base.Pos, ir.OANDAND, eqlen, eqmem) |
| } else { |
| // len(left) != len(right) || !memequal(left, right, len) |
| eqlen.SetOp(ir.ONE) |
| r = ir.NewLogicalExpr(base.Pos, ir.OOROR, eqlen, ir.NewUnaryExpr(base.Pos, ir.ONOT, eqmem)) |
| } |
| } else { |
| // sys_cmpstring(s1, s2) :: 0 |
| r = mkcall("cmpstring", types.Types[types.TINT], init, typecheck.Conv(n.X, types.Types[types.TSTRING]), typecheck.Conv(n.Y, types.Types[types.TSTRING])) |
| r = ir.NewBinaryExpr(base.Pos, n.Op(), r, ir.NewInt(0)) |
| } |
| |
| return finishCompare(n, r, init) |
| } |
| |
| // The result of finishCompare MUST be assigned back to n, e.g. |
| // n.Left = finishCompare(n.Left, x, r, init) |
| func finishCompare(n *ir.BinaryExpr, r ir.Node, init *ir.Nodes) ir.Node { |
| r = typecheck.Expr(r) |
| r = typecheck.Conv(r, n.Type()) |
| r = walkExpr(r, init) |
| return r |
| } |
| |
| func eqFor(t *types.Type) (n ir.Node, needsize bool) { |
| // Should only arrive here with large memory or |
| // a struct/array containing a non-memory field/element. |
| // Small memory is handled inline, and single non-memory |
| // is handled by walkCompare. |
| switch a, _ := types.AlgType(t); a { |
| case types.AMEM: |
| n := typecheck.LookupRuntime("memequal") |
| n = typecheck.SubstArgTypes(n, t, t) |
| return n, true |
| case types.ASPECIAL: |
| sym := reflectdata.TypeSymPrefix(".eq", t) |
| // TODO(austin): This creates an ir.Name with a nil Func. |
| n := typecheck.NewName(sym) |
| ir.MarkFunc(n) |
| n.SetType(types.NewSignature(types.NoPkg, nil, nil, []*types.Field{ |
| types.NewField(base.Pos, nil, types.NewPtr(t)), |
| types.NewField(base.Pos, nil, types.NewPtr(t)), |
| }, []*types.Field{ |
| types.NewField(base.Pos, nil, types.Types[types.TBOOL]), |
| })) |
| return n, false |
| } |
| base.Fatalf("eqFor %v", t) |
| return nil, false |
| } |
| |
| // brcom returns !(op). |
| // For example, brcom(==) is !=. |
| func brcom(op ir.Op) ir.Op { |
| switch op { |
| case ir.OEQ: |
| return ir.ONE |
| case ir.ONE: |
| return ir.OEQ |
| case ir.OLT: |
| return ir.OGE |
| case ir.OGT: |
| return ir.OLE |
| case ir.OLE: |
| return ir.OGT |
| case ir.OGE: |
| return ir.OLT |
| } |
| base.Fatalf("brcom: no com for %v\n", op) |
| return op |
| } |
| |
| // brrev returns reverse(op). |
| // For example, Brrev(<) is >. |
| func brrev(op ir.Op) ir.Op { |
| switch op { |
| case ir.OEQ: |
| return ir.OEQ |
| case ir.ONE: |
| return ir.ONE |
| case ir.OLT: |
| return ir.OGT |
| case ir.OGT: |
| return ir.OLT |
| case ir.OLE: |
| return ir.OGE |
| case ir.OGE: |
| return ir.OLE |
| } |
| base.Fatalf("brrev: no rev for %v\n", op) |
| return op |
| } |
| |
| func tracecmpArg(n ir.Node, t *types.Type, init *ir.Nodes) ir.Node { |
| // Ugly hack to avoid "constant -1 overflows uintptr" errors, etc. |
| if n.Op() == ir.OLITERAL && n.Type().IsSigned() && ir.Int64Val(n) < 0 { |
| n = copyExpr(n, n.Type(), init) |
| } |
| |
| return typecheck.Conv(n, t) |
| } |
| |
| // canMergeLoads reports whether the backend optimization passes for |
| // the current architecture can combine adjacent loads into a single |
| // larger, possibly unaligned, load. Note that currently the |
| // optimizations must be able to handle little endian byte order. |
| func canMergeLoads() bool { |
| switch ssagen.Arch.LinkArch.Family { |
| case sys.ARM64, sys.AMD64, sys.I386, sys.S390X: |
| return true |
| case sys.PPC64: |
| // Load combining only supported on ppc64le. |
| return ssagen.Arch.LinkArch.ByteOrder == binary.LittleEndian |
| } |
| return false |
| } |