runtime: split PCs out of scase
Per-case PCs are only needed for race detector builds, so this allows
skipping allocating stack space for them for non-race builds.
It's possible to arrange the PCs and order arrays consecutively in
memory so that we could just reuse the order0 pointer to identify
both. However, there's more risk of that silently going wrong, so this
commit passes them as separate arguments for now. We can revisit this
in the future.
Updates #40410.
Change-Id: I8468bc25749e559891cb0cb007d1cc4a40fdd0f8
Reviewed-on: https://go-review.googlesource.com/c/go/+/245124
Reviewed-by: Keith Randall <khr@golang.org>
diff --git a/src/cmd/compile/internal/gc/builtin.go b/src/cmd/compile/internal/gc/builtin.go
index 2cf2f46..eafdb0e 100644
--- a/src/cmd/compile/internal/gc/builtin.go
+++ b/src/cmd/compile/internal/gc/builtin.go
@@ -126,74 +126,74 @@
{"selectnbsend", funcTag, 94},
{"selectnbrecv", funcTag, 95},
{"selectnbrecv2", funcTag, 97},
- {"selectsetpc", funcTag, 62},
- {"selectgo", funcTag, 98},
+ {"selectsetpc", funcTag, 98},
+ {"selectgo", funcTag, 99},
{"block", funcTag, 9},
- {"makeslice", funcTag, 99},
- {"makeslice64", funcTag, 100},
- {"makeslicecopy", funcTag, 101},
- {"growslice", funcTag, 103},
- {"memmove", funcTag, 104},
- {"memclrNoHeapPointers", funcTag, 105},
- {"memclrHasPointers", funcTag, 105},
- {"memequal", funcTag, 106},
- {"memequal0", funcTag, 107},
- {"memequal8", funcTag, 107},
- {"memequal16", funcTag, 107},
- {"memequal32", funcTag, 107},
- {"memequal64", funcTag, 107},
- {"memequal128", funcTag, 107},
- {"f32equal", funcTag, 108},
- {"f64equal", funcTag, 108},
- {"c64equal", funcTag, 108},
- {"c128equal", funcTag, 108},
- {"strequal", funcTag, 108},
- {"interequal", funcTag, 108},
- {"nilinterequal", funcTag, 108},
- {"memhash", funcTag, 109},
- {"memhash0", funcTag, 110},
- {"memhash8", funcTag, 110},
- {"memhash16", funcTag, 110},
- {"memhash32", funcTag, 110},
- {"memhash64", funcTag, 110},
- {"memhash128", funcTag, 110},
- {"f32hash", funcTag, 110},
- {"f64hash", funcTag, 110},
- {"c64hash", funcTag, 110},
- {"c128hash", funcTag, 110},
- {"strhash", funcTag, 110},
- {"interhash", funcTag, 110},
- {"nilinterhash", funcTag, 110},
- {"int64div", funcTag, 111},
- {"uint64div", funcTag, 112},
- {"int64mod", funcTag, 111},
- {"uint64mod", funcTag, 112},
- {"float64toint64", funcTag, 113},
- {"float64touint64", funcTag, 114},
- {"float64touint32", funcTag, 115},
- {"int64tofloat64", funcTag, 116},
- {"uint64tofloat64", funcTag, 117},
- {"uint32tofloat64", funcTag, 118},
- {"complex128div", funcTag, 119},
- {"racefuncenter", funcTag, 120},
+ {"makeslice", funcTag, 100},
+ {"makeslice64", funcTag, 101},
+ {"makeslicecopy", funcTag, 102},
+ {"growslice", funcTag, 104},
+ {"memmove", funcTag, 105},
+ {"memclrNoHeapPointers", funcTag, 106},
+ {"memclrHasPointers", funcTag, 106},
+ {"memequal", funcTag, 107},
+ {"memequal0", funcTag, 108},
+ {"memequal8", funcTag, 108},
+ {"memequal16", funcTag, 108},
+ {"memequal32", funcTag, 108},
+ {"memequal64", funcTag, 108},
+ {"memequal128", funcTag, 108},
+ {"f32equal", funcTag, 109},
+ {"f64equal", funcTag, 109},
+ {"c64equal", funcTag, 109},
+ {"c128equal", funcTag, 109},
+ {"strequal", funcTag, 109},
+ {"interequal", funcTag, 109},
+ {"nilinterequal", funcTag, 109},
+ {"memhash", funcTag, 110},
+ {"memhash0", funcTag, 111},
+ {"memhash8", funcTag, 111},
+ {"memhash16", funcTag, 111},
+ {"memhash32", funcTag, 111},
+ {"memhash64", funcTag, 111},
+ {"memhash128", funcTag, 111},
+ {"f32hash", funcTag, 111},
+ {"f64hash", funcTag, 111},
+ {"c64hash", funcTag, 111},
+ {"c128hash", funcTag, 111},
+ {"strhash", funcTag, 111},
+ {"interhash", funcTag, 111},
+ {"nilinterhash", funcTag, 111},
+ {"int64div", funcTag, 112},
+ {"uint64div", funcTag, 113},
+ {"int64mod", funcTag, 112},
+ {"uint64mod", funcTag, 113},
+ {"float64toint64", funcTag, 114},
+ {"float64touint64", funcTag, 115},
+ {"float64touint32", funcTag, 116},
+ {"int64tofloat64", funcTag, 117},
+ {"uint64tofloat64", funcTag, 118},
+ {"uint32tofloat64", funcTag, 119},
+ {"complex128div", funcTag, 120},
+ {"racefuncenter", funcTag, 121},
{"racefuncenterfp", funcTag, 9},
{"racefuncexit", funcTag, 9},
- {"raceread", funcTag, 120},
- {"racewrite", funcTag, 120},
- {"racereadrange", funcTag, 121},
- {"racewriterange", funcTag, 121},
- {"msanread", funcTag, 121},
- {"msanwrite", funcTag, 121},
- {"checkptrAlignment", funcTag, 122},
- {"checkptrArithmetic", funcTag, 124},
- {"libfuzzerTraceCmp1", funcTag, 126},
- {"libfuzzerTraceCmp2", funcTag, 128},
- {"libfuzzerTraceCmp4", funcTag, 129},
- {"libfuzzerTraceCmp8", funcTag, 130},
- {"libfuzzerTraceConstCmp1", funcTag, 126},
- {"libfuzzerTraceConstCmp2", funcTag, 128},
- {"libfuzzerTraceConstCmp4", funcTag, 129},
- {"libfuzzerTraceConstCmp8", funcTag, 130},
+ {"raceread", funcTag, 121},
+ {"racewrite", funcTag, 121},
+ {"racereadrange", funcTag, 122},
+ {"racewriterange", funcTag, 122},
+ {"msanread", funcTag, 122},
+ {"msanwrite", funcTag, 122},
+ {"checkptrAlignment", funcTag, 123},
+ {"checkptrArithmetic", funcTag, 125},
+ {"libfuzzerTraceCmp1", funcTag, 127},
+ {"libfuzzerTraceCmp2", funcTag, 129},
+ {"libfuzzerTraceCmp4", funcTag, 130},
+ {"libfuzzerTraceCmp8", funcTag, 131},
+ {"libfuzzerTraceConstCmp1", funcTag, 127},
+ {"libfuzzerTraceConstCmp2", funcTag, 129},
+ {"libfuzzerTraceConstCmp4", funcTag, 130},
+ {"libfuzzerTraceConstCmp8", funcTag, 131},
{"x86HasPOPCNT", varTag, 6},
{"x86HasSSE41", varTag, 6},
{"x86HasFMA", varTag, 6},
@@ -202,7 +202,7 @@
}
func runtimeTypes() []*types.Type {
- var typs [131]*types.Type
+ var typs [132]*types.Type
typs[0] = types.Bytetype
typs[1] = types.NewPtr(typs[0])
typs[2] = types.Types[TANY]
@@ -301,38 +301,39 @@
typs[95] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[84])}, []*Node{anonfield(typs[6])})
typs[96] = types.NewPtr(typs[6])
typs[97] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[96]), anonfield(typs[84])}, []*Node{anonfield(typs[6])})
- typs[98] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[1]), anonfield(typs[15])}, []*Node{anonfield(typs[15]), anonfield(typs[6])})
- typs[99] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15])}, []*Node{anonfield(typs[7])})
- typs[100] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[22]), anonfield(typs[22])}, []*Node{anonfield(typs[7])})
- typs[101] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15]), anonfield(typs[7])}, []*Node{anonfield(typs[7])})
- typs[102] = types.NewSlice(typs[2])
- typs[103] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[102]), anonfield(typs[15])}, []*Node{anonfield(typs[102])})
- typs[104] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[5])}, nil)
- typs[105] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5])}, nil)
- typs[106] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[5])}, []*Node{anonfield(typs[6])})
- typs[107] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3])}, []*Node{anonfield(typs[6])})
- typs[108] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[7])}, []*Node{anonfield(typs[6])})
- typs[109] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5]), anonfield(typs[5])}, []*Node{anonfield(typs[5])})
- typs[110] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5])}, []*Node{anonfield(typs[5])})
- typs[111] = functype(nil, []*Node{anonfield(typs[22]), anonfield(typs[22])}, []*Node{anonfield(typs[22])})
- typs[112] = functype(nil, []*Node{anonfield(typs[24]), anonfield(typs[24])}, []*Node{anonfield(typs[24])})
- typs[113] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[22])})
- typs[114] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[24])})
- typs[115] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[65])})
- typs[116] = functype(nil, []*Node{anonfield(typs[22])}, []*Node{anonfield(typs[20])})
- typs[117] = functype(nil, []*Node{anonfield(typs[24])}, []*Node{anonfield(typs[20])})
- typs[118] = functype(nil, []*Node{anonfield(typs[65])}, []*Node{anonfield(typs[20])})
- typs[119] = functype(nil, []*Node{anonfield(typs[26]), anonfield(typs[26])}, []*Node{anonfield(typs[26])})
- typs[120] = functype(nil, []*Node{anonfield(typs[5])}, nil)
- typs[121] = functype(nil, []*Node{anonfield(typs[5]), anonfield(typs[5])}, nil)
- typs[122] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[1]), anonfield(typs[5])}, nil)
- typs[123] = types.NewSlice(typs[7])
- typs[124] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[123])}, nil)
- typs[125] = types.Types[TUINT8]
- typs[126] = functype(nil, []*Node{anonfield(typs[125]), anonfield(typs[125])}, nil)
- typs[127] = types.Types[TUINT16]
- typs[128] = functype(nil, []*Node{anonfield(typs[127]), anonfield(typs[127])}, nil)
- typs[129] = functype(nil, []*Node{anonfield(typs[65]), anonfield(typs[65])}, nil)
- typs[130] = functype(nil, []*Node{anonfield(typs[24]), anonfield(typs[24])}, nil)
+ typs[98] = functype(nil, []*Node{anonfield(typs[63])}, nil)
+ typs[99] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[1]), anonfield(typs[63]), anonfield(typs[15])}, []*Node{anonfield(typs[15]), anonfield(typs[6])})
+ typs[100] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15])}, []*Node{anonfield(typs[7])})
+ typs[101] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[22]), anonfield(typs[22])}, []*Node{anonfield(typs[7])})
+ typs[102] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15]), anonfield(typs[7])}, []*Node{anonfield(typs[7])})
+ typs[103] = types.NewSlice(typs[2])
+ typs[104] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[103]), anonfield(typs[15])}, []*Node{anonfield(typs[103])})
+ typs[105] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[5])}, nil)
+ typs[106] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5])}, nil)
+ typs[107] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[5])}, []*Node{anonfield(typs[6])})
+ typs[108] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3])}, []*Node{anonfield(typs[6])})
+ typs[109] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[7])}, []*Node{anonfield(typs[6])})
+ typs[110] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5]), anonfield(typs[5])}, []*Node{anonfield(typs[5])})
+ typs[111] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5])}, []*Node{anonfield(typs[5])})
+ typs[112] = functype(nil, []*Node{anonfield(typs[22]), anonfield(typs[22])}, []*Node{anonfield(typs[22])})
+ typs[113] = functype(nil, []*Node{anonfield(typs[24]), anonfield(typs[24])}, []*Node{anonfield(typs[24])})
+ typs[114] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[22])})
+ typs[115] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[24])})
+ typs[116] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[65])})
+ typs[117] = functype(nil, []*Node{anonfield(typs[22])}, []*Node{anonfield(typs[20])})
+ typs[118] = functype(nil, []*Node{anonfield(typs[24])}, []*Node{anonfield(typs[20])})
+ typs[119] = functype(nil, []*Node{anonfield(typs[65])}, []*Node{anonfield(typs[20])})
+ typs[120] = functype(nil, []*Node{anonfield(typs[26]), anonfield(typs[26])}, []*Node{anonfield(typs[26])})
+ typs[121] = functype(nil, []*Node{anonfield(typs[5])}, nil)
+ typs[122] = functype(nil, []*Node{anonfield(typs[5]), anonfield(typs[5])}, nil)
+ typs[123] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[1]), anonfield(typs[5])}, nil)
+ typs[124] = types.NewSlice(typs[7])
+ typs[125] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[124])}, nil)
+ typs[126] = types.Types[TUINT8]
+ typs[127] = functype(nil, []*Node{anonfield(typs[126]), anonfield(typs[126])}, nil)
+ typs[128] = types.Types[TUINT16]
+ typs[129] = functype(nil, []*Node{anonfield(typs[128]), anonfield(typs[128])}, nil)
+ typs[130] = functype(nil, []*Node{anonfield(typs[65]), anonfield(typs[65])}, nil)
+ typs[131] = functype(nil, []*Node{anonfield(typs[24]), anonfield(typs[24])}, nil)
return typs[:]
}
diff --git a/src/cmd/compile/internal/gc/builtin/runtime.go b/src/cmd/compile/internal/gc/builtin/runtime.go
index 0044827..25f86ef 100644
--- a/src/cmd/compile/internal/gc/builtin/runtime.go
+++ b/src/cmd/compile/internal/gc/builtin/runtime.go
@@ -169,8 +169,8 @@
func selectnbrecv(elem *any, hchan <-chan any) bool
func selectnbrecv2(elem *any, received *bool, hchan <-chan any) bool
-func selectsetpc(cas *byte)
-func selectgo(cas0 *byte, order0 *byte, ncases int) (int, bool)
+func selectsetpc(pc *uintptr)
+func selectgo(cas0 *byte, order0 *byte, pc0 *uintptr, ncases int) (int, bool)
func block()
func makeslice(typ *byte, len int, cap int) unsafe.Pointer
diff --git a/src/cmd/compile/internal/gc/select.go b/src/cmd/compile/internal/gc/select.go
index eb5ff84..8eb31eb 100644
--- a/src/cmd/compile/internal/gc/select.go
+++ b/src/cmd/compile/internal/gc/select.go
@@ -271,6 +271,14 @@
r = typecheck(r, ctxStmt)
init = append(init, r)
+ var pc0, pcs *Node
+ if flag_race {
+ pcs = temp(types.NewArray(types.Types[TUINTPTR], int64(n)))
+ pc0 = typecheck(nod(OADDR, nod(OINDEX, pcs, nodintconst(0)), nil), ctxExpr)
+ } else {
+ pc0 = nodnil()
+ }
+
// register cases
for i, cas := range cases.Slice() {
setlineno(cas)
@@ -324,8 +332,8 @@
// TODO(mdempsky): There should be a cleaner way to
// handle this.
- if instrumenting {
- r = mkcall("selectsetpc", nil, nil, bytePtrToIndex(selv, int64(i)))
+ if flag_race {
+ r = mkcall("selectsetpc", nil, nil, nod(OADDR, nod(OINDEX, pcs, nodintconst(int64(i))), nil))
init = append(init, r)
}
}
@@ -337,13 +345,16 @@
r = nod(OAS2, nil, nil)
r.List.Set2(chosen, recvOK)
fn := syslook("selectgo")
- r.Rlist.Set1(mkcall1(fn, fn.Type.Results(), nil, bytePtrToIndex(selv, 0), bytePtrToIndex(order, 0), nodintconst(int64(n))))
+ r.Rlist.Set1(mkcall1(fn, fn.Type.Results(), nil, bytePtrToIndex(selv, 0), bytePtrToIndex(order, 0), pc0, nodintconst(int64(n))))
r = typecheck(r, ctxStmt)
init = append(init, r)
// selv and order are no longer alive after selectgo.
init = append(init, nod(OVARKILL, selv, nil))
init = append(init, nod(OVARKILL, order, nil))
+ if flag_race {
+ init = append(init, nod(OVARKILL, pcs, nil))
+ }
// dispatch cases
for i, cas := range cases.Slice() {
@@ -385,7 +396,6 @@
namedfield("c", types.Types[TUNSAFEPTR]),
namedfield("elem", types.Types[TUNSAFEPTR]),
namedfield("kind", types.Types[TUINT16]),
- namedfield("pc", types.Types[TUINTPTR]),
})
scase.SetNoalg(true)
}
diff --git a/src/runtime/select.go b/src/runtime/select.go
index d540dd2..d7c7d9f 100644
--- a/src/runtime/select.go
+++ b/src/runtime/select.go
@@ -29,7 +29,6 @@
c *hchan // chan
elem unsafe.Pointer // data element
kind uint16
- pc uintptr // race pc (for race detector / msan)
}
var (
@@ -37,8 +36,8 @@
chanrecvpc = funcPC(chanrecv)
)
-func selectsetpc(cas *scase) {
- cas.pc = getcallerpc()
+func selectsetpc(pc *uintptr) {
+ *pc = getcallerpc()
}
func sellock(scases []scase, lockorder []uint16) {
@@ -108,11 +107,15 @@
// Both reside on the goroutine's stack (regardless of any escaping in
// selectgo).
//
+// For race detector builds, pc0 points to an array of type
+// [ncases]uintptr (also on the stack); for other builds, it's set to
+// nil.
+//
// selectgo returns the index of the chosen scase, which matches the
// ordinal position of its respective select{recv,send,default} call.
// Also, if the chosen scase was a receive operation, it reports whether
// a value was received.
-func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) {
+func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) {
if debugSelect {
print("select: cas0=", cas0, "\n")
}
@@ -126,6 +129,21 @@
pollorder := order1[:ncases:ncases]
lockorder := order1[ncases:][:ncases:ncases]
+ // Even when raceenabled is true, there might be select
+ // statements in packages compiled without -race (e.g.,
+ // ensureSigM in runtime/signal_unix.go).
+ var pcs []uintptr
+ if raceenabled && pc0 != nil {
+ pc1 := (*[1 << 16]uintptr)(unsafe.Pointer(pc0))
+ pcs = pc1[:ncases:ncases]
+ }
+ casePC := func(casi int) uintptr {
+ if pcs == nil {
+ return 0
+ }
+ return pcs[casi]
+ }
+
var t0 int64
if blockprofilerate > 0 {
t0 = cputicks()
@@ -247,7 +265,7 @@
case caseSend:
if raceenabled {
- racereadpc(c.raceaddr(), cas.pc, chansendpc)
+ racereadpc(c.raceaddr(), casePC(casi), chansendpc)
}
if c.closed != 0 {
goto sclose
@@ -371,9 +389,9 @@
if raceenabled {
if cas.kind == caseRecv && cas.elem != nil {
- raceWriteObjectPC(c.elemtype, cas.elem, cas.pc, chanrecvpc)
+ raceWriteObjectPC(c.elemtype, cas.elem, casePC(casi), chanrecvpc)
} else if cas.kind == caseSend {
- raceReadObjectPC(c.elemtype, cas.elem, cas.pc, chansendpc)
+ raceReadObjectPC(c.elemtype, cas.elem, casePC(casi), chansendpc)
}
}
if msanenabled {
@@ -391,7 +409,7 @@
// can receive from buffer
if raceenabled {
if cas.elem != nil {
- raceWriteObjectPC(c.elemtype, cas.elem, cas.pc, chanrecvpc)
+ raceWriteObjectPC(c.elemtype, cas.elem, casePC(casi), chanrecvpc)
}
raceacquire(chanbuf(c, c.recvx))
racerelease(chanbuf(c, c.recvx))
@@ -418,7 +436,7 @@
if raceenabled {
raceacquire(chanbuf(c, c.sendx))
racerelease(chanbuf(c, c.sendx))
- raceReadObjectPC(c.elemtype, cas.elem, cas.pc, chansendpc)
+ raceReadObjectPC(c.elemtype, cas.elem, casePC(casi), chansendpc)
}
if msanenabled {
msanread(cas.elem, c.elemtype.size)
@@ -456,7 +474,7 @@
send:
// can send to a sleeping receiver (sg)
if raceenabled {
- raceReadObjectPC(c.elemtype, cas.elem, cas.pc, chansendpc)
+ raceReadObjectPC(c.elemtype, cas.elem, casePC(casi), chansendpc)
}
if msanenabled {
msanread(cas.elem, c.elemtype.size)
@@ -519,12 +537,18 @@
case selectRecv:
sel[i] = scase{kind: caseRecv, c: rc.ch, elem: rc.val}
}
- if raceenabled || msanenabled {
- selectsetpc(&sel[i])
- }
}
- return selectgo(&sel[0], &order[0], len(cases))
+ var pc0 *uintptr
+ if raceenabled {
+ pcs := make([]uintptr, len(cases))
+ for i := range pcs {
+ selectsetpc(&pcs[i])
+ }
+ pc0 = &pcs[0]
+ }
+
+ return selectgo(&sel[0], &order[0], pc0, len(cases))
}
func (q *waitq) dequeueSudoG(sgp *sudog) {
