runtime: convert common scheduler functions to Go
These are required for chans, semaphores, timers, etc.
LGTM=khr
R=golang-codereviews, khr
CC=golang-codereviews, rlh, rsc
https://golang.org/cl/123640043
diff --git a/src/pkg/runtime/asm_386.s b/src/pkg/runtime/asm_386.s
index 638d1c5..741832b 100644
--- a/src/pkg/runtime/asm_386.s
+++ b/src/pkg/runtime/asm_386.s
@@ -869,6 +869,12 @@
MOVL DX, 4(DI)
RET
+TEXT runtime·gocputicks(SB),NOSPLIT,$0-8
+ RDTSC
+ MOVL AX, ret+0(FP)
+ MOVL DX, ret+4(FP)
+ RET
+
TEXT runtime·ldt0setup(SB),NOSPLIT,$16-0
// set up ldt 7 to point at tls0
// ldt 1 would be fine on Linux, but on OS X, 7 is as low as we can go.
diff --git a/src/pkg/runtime/asm_amd64.s b/src/pkg/runtime/asm_amd64.s
index 87c2316..9fcb231 100644
--- a/src/pkg/runtime/asm_amd64.s
+++ b/src/pkg/runtime/asm_amd64.s
@@ -953,6 +953,12 @@
ADDQ DX, AX
RET
+TEXT runtime·gocputicks(SB),NOSPLIT,$0-8
+ RDTSC
+ MOVL AX, ret+0(FP)
+ MOVL DX, ret+4(FP)
+ RET
+
TEXT runtime·stackguard(SB),NOSPLIT,$0-16
MOVQ SP, DX
MOVQ DX, sp+0(FP)
diff --git a/src/pkg/runtime/asm_amd64p32.s b/src/pkg/runtime/asm_amd64p32.s
index 3e84d95..5c6a73a 100644
--- a/src/pkg/runtime/asm_amd64p32.s
+++ b/src/pkg/runtime/asm_amd64p32.s
@@ -754,6 +754,13 @@
ADDQ DX, AX
RET
+TEXT runtime·gocputicks(SB),NOSPLIT,$0-8
+ RDTSC
+ SHLQ $32, DX
+ ADDQ DX, AX
+ MOVQ AX, ret+0(FP)
+ RET
+
TEXT runtime·stackguard(SB),NOSPLIT,$0-8
MOVL SP, DX
MOVL DX, sp+0(FP)
diff --git a/src/pkg/runtime/asm_arm.s b/src/pkg/runtime/asm_arm.s
index 3e0b13e..4e46767 100644
--- a/src/pkg/runtime/asm_arm.s
+++ b/src/pkg/runtime/asm_arm.s
@@ -651,6 +651,12 @@
MOVW $0, R0
MOVW (R0), R1
+TEXT runtime·gocputicks(SB),NOSPLIT,$4-8
+ ADD $8, SP, R0
+ MOVW R0, 0(SP)
+ BL runtime·cputicks(SB)
+ RET
+
// bool armcas(int32 *val, int32 old, int32 new)
// Atomically:
// if(*val == old){
diff --git a/src/pkg/runtime/chan.goc b/src/pkg/runtime/chan.goc
index 7ddfab3..2ef1c85 100644
--- a/src/pkg/runtime/chan.goc
+++ b/src/pkg/runtime/chan.goc
@@ -119,7 +119,7 @@
USED(t);
if(!block)
return false;
- runtime·park(nil, nil, "chan send (nil chan)");
+ runtime·park(nil, nil, runtime·gostringnocopy((byte*)"chan send (nil chan)"));
return false; // not reached
}
@@ -171,7 +171,7 @@
mysg.selectdone = nil;
g->param = nil;
enqueue(&c->sendq, &mysg);
- runtime·parkunlock(&c->lock, "chan send");
+ runtime·parkunlock(&c->lock, runtime·gostringnocopy((byte*)"chan send"));
if(g->param == nil) {
runtime·lock(&c->lock);
@@ -198,7 +198,7 @@
mysg.elem = nil;
mysg.selectdone = nil;
enqueue(&c->sendq, &mysg);
- runtime·parkunlock(&c->lock, "chan send");
+ runtime·parkunlock(&c->lock, runtime·gostringnocopy((byte*)"chan send"));
runtime·lock(&c->lock);
goto asynch;
@@ -251,7 +251,7 @@
USED(t);
if(!block)
return false;
- runtime·park(nil, nil, "chan receive (nil chan)");
+ runtime·park(nil, nil, runtime·gostringnocopy((byte*)"chan receive (nil chan)"));
return false; // not reached
}
@@ -298,7 +298,7 @@
mysg.selectdone = nil;
g->param = nil;
enqueue(&c->recvq, &mysg);
- runtime·parkunlock(&c->lock, "chan receive");
+ runtime·parkunlock(&c->lock, runtime·gostringnocopy((byte*)"chan receive"));
if(g->param == nil) {
runtime·lock(&c->lock);
@@ -328,7 +328,7 @@
mysg.elem = nil;
mysg.selectdone = nil;
enqueue(&c->recvq, &mysg);
- runtime·parkunlock(&c->lock, "chan receive");
+ runtime·parkunlock(&c->lock, runtime·gostringnocopy((byte*)"chan receive"));
runtime·lock(&c->lock);
goto asynch;
@@ -658,7 +658,7 @@
}
func block() {
- runtime·park(nil, nil, "select (no cases)"); // forever
+ runtime·park(nil, nil, runtime·gostringnocopy((byte*)"select (no cases)")); // forever
}
static void* selectgo(Select**);
@@ -826,7 +826,7 @@
}
g->param = nil;
- runtime·park(selparkcommit, sel, "select");
+ runtime·park(selparkcommit, sel, runtime·gostringnocopy((byte*)"select"));
sellock(sel);
sg = g->param;
diff --git a/src/pkg/runtime/chan.h b/src/pkg/runtime/chan.h
index e6e6bac..5ac39ca 100644
--- a/src/pkg/runtime/chan.h
+++ b/src/pkg/runtime/chan.h
@@ -18,6 +18,7 @@
SudoG* link;
byte* elem; // data element
int64 releasetime;
+ int32 nrelease; // -1 for acquire
};
struct WaitQ
diff --git a/src/pkg/runtime/heapdump.c b/src/pkg/runtime/heapdump.c
index a2d12ad..3fc54ae 100644
--- a/src/pkg/runtime/heapdump.c
+++ b/src/pkg/runtime/heapdump.c
@@ -408,7 +408,7 @@
dumpbool(gp->issystem);
dumpbool(gp->isbackground);
dumpint(gp->waitsince);
- dumpcstr(gp->waitreason);
+ dumpstr(gp->waitreason);
dumpint((uintptr)gp->sched.ctxt);
dumpint((uintptr)gp->m);
dumpint((uintptr)gp->defer);
@@ -760,7 +760,7 @@
// Call dump routine on M stack.
g->status = Gwaiting;
- g->waitreason = "dumping heap";
+ g->waitreason = runtime·gostringnocopy((byte*)"dumping heap");
runtime·mcall(mdump);
// Reset dump file.
diff --git a/src/pkg/runtime/malloc.h b/src/pkg/runtime/malloc.h
index a700956..4485100 100644
--- a/src/pkg/runtime/malloc.h
+++ b/src/pkg/runtime/malloc.h
@@ -318,6 +318,8 @@
uintptr size; // total size of stacks in list
};
+typedef struct SudoG SudoG;
+
// Per-thread (in Go, per-P) cache for small objects.
// No locking needed because it is per-thread (per-P).
struct MCache
@@ -335,6 +337,8 @@
StackFreeList stackcache[NumStackOrders];
+ SudoG* sudogcache;
+
void* gcworkbuf;
// Local allocator stats, flushed during GC.
diff --git a/src/pkg/runtime/mgc0.c b/src/pkg/runtime/mgc0.c
index 14743c2..2d378e2 100644
--- a/src/pkg/runtime/mgc0.c
+++ b/src/pkg/runtime/mgc0.c
@@ -107,6 +107,7 @@
if(c != nil) {
c->tiny = nil;
c->tinysize = 0;
+ c->sudogcache = nil;
}
// clear defer pools
for(i=0; i<nelem(p->deferpool); i++)
@@ -1110,7 +1111,7 @@
}
sweep.parked = true;
g->isbackground = true;
- runtime·parkunlock(&gclock, "GC sweep wait");
+ runtime·parkunlock(&gclock, runtime·gostringnocopy((byte*)"GC sweep wait"));
g->isbackground = false;
}
}
@@ -1374,7 +1375,7 @@
// switch to g0, call gc(&a), then switch back
g->param = &a;
g->status = Gwaiting;
- g->waitreason = "garbage collection";
+ g->waitreason = runtime·gostringnocopy((byte*)"garbage collection");
runtime·mcall(mgc);
}
@@ -1409,7 +1410,7 @@
gp = g->m->curg;
gp->status = Gwaiting;
- gp->waitreason = "garbage collection";
+ gp->waitreason = runtime·gostringnocopy((byte*)"garbage collection");
a.start_time = (uint64)(g->m->scalararg[0]) | ((uint64)(g->m->scalararg[1]) << 32);
a.eagersweep = g->m->scalararg[2];
@@ -1663,7 +1664,7 @@
if(fb == nil) {
runtime·fingwait = true;
g->isbackground = true;
- runtime·parkunlock(&finlock, "finalizer wait");
+ runtime·parkunlock(&finlock, runtime·gostringnocopy((byte*)"finalizer wait"));
g->isbackground = false;
continue;
}
diff --git a/src/pkg/runtime/netpoll.goc b/src/pkg/runtime/netpoll.goc
index eb7f887..bbb3d06 100644
--- a/src/pkg/runtime/netpoll.goc
+++ b/src/pkg/runtime/netpoll.goc
@@ -346,7 +346,7 @@
// this is necessary because runtime_pollUnblock/runtime_pollSetDeadline/deadlineimpl
// do the opposite: store to closing/rd/wd, membarrier, load of rg/wg
if(waitio || checkerr(pd, mode) == 0)
- runtime·park((bool(*)(G*, void*))blockcommit, gpp, "IO wait");
+ runtime·park((bool(*)(G*, void*))blockcommit, gpp, runtime·gostringnocopy((byte*)"IO wait"));
// be careful to not lose concurrent READY notification
old = runtime·xchgp(gpp, nil);
if(old > WAIT)
diff --git a/src/pkg/runtime/proc.c b/src/pkg/runtime/proc.c
index 1e0cd38..5ff38fb 100644
--- a/src/pkg/runtime/proc.c
+++ b/src/pkg/runtime/proc.c
@@ -113,7 +113,7 @@
static void incidlelocked(int32);
static void checkdead(void);
static void exitsyscall0(G*);
-static void park0(G*);
+void runtime·park_m(G*);
static void goexit0(G*);
static void gfput(P*, G*);
static G* gfget(P*);
@@ -265,7 +265,7 @@
// let the other goroutine finish printing the panic trace.
// Once it does, it will exit. See issue 3934.
if(runtime·panicking)
- runtime·park(nil, nil, "panicwait");
+ runtime·park(nil, nil, runtime·gostringnocopy((byte*)"panicwait"));
runtime·exit(0);
for(;;)
@@ -275,30 +275,30 @@
void
runtime·goroutineheader(G *gp)
{
- int8 *status;
+ String status;
int64 waitfor;
switch(gp->status) {
case Gidle:
- status = "idle";
+ status = runtime·gostringnocopy((byte*)"idle");
break;
case Grunnable:
- status = "runnable";
+ status = runtime·gostringnocopy((byte*)"runnable");
break;
case Grunning:
- status = "running";
+ status = runtime·gostringnocopy((byte*)"running");
break;
case Gsyscall:
- status = "syscall";
+ status = runtime·gostringnocopy((byte*)"syscall");
break;
case Gwaiting:
- if(gp->waitreason)
+ if(gp->waitreason.str != nil)
status = gp->waitreason;
else
- status = "waiting";
+ status = runtime·gostringnocopy((byte*)"waiting");
break;
default:
- status = "???";
+ status = runtime·gostringnocopy((byte*)"???");
break;
}
@@ -307,7 +307,7 @@
if((gp->status == Gwaiting || gp->status == Gsyscall) && gp->waitsince != 0)
waitfor = (runtime·nanotime() - gp->waitsince) / (60LL*1000*1000*1000);
- runtime·printf("goroutine %D [%s", gp->goid, status);
+ runtime·printf("goroutine %D [%S", gp->goid, status);
if(waitfor >= 1)
runtime·printf(", %D minutes", waitfor);
if(gp->lockedm != nil)
@@ -401,6 +401,16 @@
g->stackguard0 = StackPreempt;
}
+void
+runtime·ready_m(void)
+{
+ G *gp;
+
+ gp = g->m->ptrarg[0];
+ g->m->ptrarg[0] = nil;
+ runtime·ready(gp);
+}
+
int32
runtime·gcprocs(void)
{
@@ -1401,18 +1411,18 @@
// Puts the current goroutine into a waiting state and calls unlockf.
// If unlockf returns false, the goroutine is resumed.
void
-runtime·park(bool(*unlockf)(G*, void*), void *lock, int8 *reason)
+runtime·park(bool(*unlockf)(G*, void*), void *lock, String reason)
{
if(g->status != Grunning)
runtime·throw("bad g status");
g->m->waitlock = lock;
g->m->waitunlockf = unlockf;
g->waitreason = reason;
- runtime·mcall(park0);
+ runtime·mcall(runtime·park_m);
}
-static bool
-parkunlock(G *gp, void *lock)
+bool
+runtime·parkunlock_c(G *gp, void *lock)
{
USED(gp);
runtime·unlock(lock);
@@ -1422,14 +1432,14 @@
// Puts the current goroutine into a waiting state and unlocks the lock.
// The goroutine can be made runnable again by calling runtime·ready(gp).
void
-runtime·parkunlock(Lock *lock, int8 *reason)
+runtime·parkunlock(Lock *lock, String reason)
{
- runtime·park(parkunlock, lock, reason);
+ runtime·park(runtime·parkunlock_c, lock, reason);
}
// runtime·park continuation on g0.
-static void
-park0(G *gp)
+void
+runtime·park_m(G *gp)
{
bool ok;
@@ -1499,7 +1509,8 @@
gp->panic = nil; // non-nil for Goexit during panic. points at stack-allocated data.
gp->writenbuf = 0;
gp->writebuf = nil;
- gp->waitreason = nil;
+ gp->waitreason.str = nil;
+ gp->waitreason.len = 0;
gp->param = nil;
dropg();
@@ -2819,7 +2830,7 @@
gp = runtime·allg[gi];
mp = gp->m;
lockedm = gp->lockedm;
- runtime·printf(" G%D: status=%d(%s) m=%d lockedm=%d\n",
+ runtime·printf(" G%D: status=%d(%S) m=%d lockedm=%d\n",
gp->goid, gp->status, gp->waitreason, mp ? mp->id : -1,
lockedm ? lockedm->id : -1);
}
diff --git a/src/pkg/runtime/proc.go b/src/pkg/runtime/proc.go
index 1b586e8..7a3864f 100644
--- a/src/pkg/runtime/proc.go
+++ b/src/pkg/runtime/proc.go
@@ -4,8 +4,79 @@
package runtime
+import "unsafe"
+
+const (
+ gStatusidle = iota
+ gStatusRunnable
+ gStatusRunning
+ gStatusSyscall
+ gStatusWaiting
+ gStatusMoribundUnused
+ gStatusDead
+)
+
+var parkunlock_c byte
+
// Gosched yields the processor, allowing other goroutines to run. It does not
// suspend the current goroutine, so execution resumes automatically.
func Gosched() {
mcall(&gosched_m)
}
+
+// Puts the current goroutine into a waiting state and calls unlockf.
+// If unlockf returns false, the goroutine is resumed.
+func gopark(unlockf unsafe.Pointer, lock unsafe.Pointer, reason string) {
+ mp := acquirem()
+ gp := mp.curg
+ if gp.status != gStatusRunning {
+ gothrow("gopark: bad g status")
+ }
+ mp.waitlock = lock
+ mp.waitunlockf = *(*func(*g, unsafe.Pointer) uint8)(unsafe.Pointer(&unlockf))
+ gp.waitreason = reason
+ releasem(mp)
+ // can't do anything that might move the G between Ms here.
+ mcall(&park_m)
+}
+
+// Puts the current goroutine into a waiting state and unlocks the lock.
+// The goroutine can be made runnable again by calling goready(gp).
+func goparkunlock(lock *lock, reason string) {
+ gopark(unsafe.Pointer(&parkunlock_c), unsafe.Pointer(lock), reason)
+}
+
+func goready(gp *g) {
+ mp := acquirem()
+ mp.ptrarg[0] = unsafe.Pointer(gp)
+ onM(&ready_m)
+ releasem(mp)
+}
+
+func goblockevent(cycles int64, skip int32) {
+ // TODO: convert to Go when we do mprof.goc
+ mp := acquirem()
+ mp.scalararg[0] = uint(uint32(cycles))
+ mp.scalararg[1] = uint(cycles >> 32)
+ mp.scalararg[2] = uint(skip)
+ onM(&blockevent_m)
+ releasem(mp)
+}
+
+//go:nosplit
+func acquireSudog() *sudog {
+ c := gomcache()
+ s := c.sudogcache
+ if s != nil {
+ c.sudogcache = s.link
+ return s
+ }
+ return new(sudog)
+}
+
+//go:nosplit
+func releaseSudog(s *sudog) {
+ c := gomcache()
+ s.link = c.sudogcache
+ c.sudogcache = s
+}
diff --git a/src/pkg/runtime/runtime.h b/src/pkg/runtime/runtime.h
index 867da3f..2c004b4 100644
--- a/src/pkg/runtime/runtime.h
+++ b/src/pkg/runtime/runtime.h
@@ -267,7 +267,7 @@
int16 status;
int64 goid;
int64 waitsince; // approx time when the G become blocked
- int8* waitreason; // if status==Gwaiting
+ String waitreason; // if status==Gwaiting
G* schedlink;
bool ispanic;
bool issystem; // do not output in stack dump
@@ -922,9 +922,9 @@
void runtime·gosched(void);
void runtime·gosched_m(G*);
void runtime·schedtrace(bool);
-void runtime·park(bool(*)(G*, void*), void*, int8*);
-void runtime·parkunlock(Lock*, int8*);
-void runtime·tsleep(int64, int8*);
+void runtime·park(bool(*)(G*, void*), void*, String);
+void runtime·parkunlock(Lock*, String);
+void runtime·tsleep(int64, String);
M* runtime·newm(void);
void runtime·goexit(void);
void runtime·asmcgocall(void (*fn)(void*), void*);
diff --git a/src/pkg/runtime/sema.goc b/src/pkg/runtime/sema.goc
index 59a890c..7b1f8f2 100644
--- a/src/pkg/runtime/sema.goc
+++ b/src/pkg/runtime/sema.goc
@@ -137,7 +137,7 @@
// Any semrelease after the cansemacquire knows we're waiting
// (we set nwait above), so go to sleep.
semqueue(root, addr, &s);
- runtime·parkunlock(&root->lock, "semacquire");
+ runtime·parkunlock(&root->lock, runtime·gostringnocopy((byte*)"semacquire"));
if(cansemacquire(addr)) {
if(t0)
runtime·blockevent(s.releasetime - t0, 3);
@@ -254,7 +254,7 @@
else
s->tail->next = &w;
s->tail = &w;
- runtime·parkunlock(&s->lock, "semacquire");
+ runtime·parkunlock(&s->lock, runtime·gostringnocopy((byte*)"semacquire"));
if(t0)
runtime·blockevent(w.releasetime - t0, 2);
}
@@ -288,7 +288,7 @@
else
s->tail->next = &w;
s->tail = &w;
- runtime·parkunlock(&s->lock, "semarelease");
+ runtime·parkunlock(&s->lock, runtime·gostringnocopy((byte*)"semarelease"));
} else
runtime·unlock(&s->lock);
}
diff --git a/src/pkg/runtime/stack.c b/src/pkg/runtime/stack.c
index f21d544..aeb5fb7 100644
--- a/src/pkg/runtime/stack.c
+++ b/src/pkg/runtime/stack.c
@@ -343,7 +343,7 @@
gp->sched.ret = g->m->cret;
g->m->cret = 0; // drop reference
gp->status = Gwaiting;
- gp->waitreason = "stack unsplit";
+ gp->waitreason = runtime·gostringnocopy((byte*)"stack unsplit");
if(argsize > 0) {
sp -= argsize;
@@ -860,7 +860,7 @@
g->m->morebuf.lr = (uintptr)nil;
g->m->morebuf.sp = (uintptr)nil;
gp->status = Gwaiting;
- gp->waitreason = "stack growth";
+ gp->waitreason = runtime·gostringnocopy((byte*)"stack growth");
newstackcall = framesize==1;
if(newstackcall)
framesize = 0;
diff --git a/src/pkg/runtime/stubs.go b/src/pkg/runtime/stubs.go
index 39244ef..f3ac783 100644
--- a/src/pkg/runtime/stubs.go
+++ b/src/pkg/runtime/stubs.go
@@ -41,6 +41,7 @@
}
// in stubs.goc
+func getg() *g
func acquirem() *m
func releasem(mp *m)
func gomcache() *mcache
@@ -69,7 +70,10 @@
markallocated_m,
unrollgcprog_m,
unrollgcproginplace_m,
- gosched_m mFunction
+ gosched_m,
+ ready_m,
+ park_m,
+ blockevent_m mFunction
)
// memclr clears n bytes starting at ptr.
@@ -163,3 +167,5 @@
// gopersistentalloc allocates a permanent (not garbage collected)
// memory region of size n. Use wisely!
func gopersistentalloc(n uintptr) unsafe.Pointer
+
+func gocputicks() int64
diff --git a/src/pkg/runtime/stubs.goc b/src/pkg/runtime/stubs.goc
index e646b55..74762da 100644
--- a/src/pkg/runtime/stubs.goc
+++ b/src/pkg/runtime/stubs.goc
@@ -69,6 +69,11 @@
}
#pragma textflag NOSPLIT
+func runtime·getg() (ret *G) {
+ ret = g;
+}
+
+#pragma textflag NOSPLIT
func runtime·acquirem() (ret *M) {
ret = g->m;
ret->locks++;
diff --git a/src/pkg/runtime/time.goc b/src/pkg/runtime/time.goc
index fa25671..f9126bd 100644
--- a/src/pkg/runtime/time.goc
+++ b/src/pkg/runtime/time.goc
@@ -36,7 +36,7 @@
// Sleep puts the current goroutine to sleep for at least ns nanoseconds.
func Sleep(ns int64) {
- runtime·tsleep(ns, "sleep");
+ runtime·tsleep(ns, runtime·gostringnocopy((byte*)"sleep"));
}
// startTimer adds t to the timer heap.
@@ -81,7 +81,7 @@
// Put the current goroutine to sleep for ns nanoseconds.
void
-runtime·tsleep(int64 ns, int8 *reason)
+runtime·tsleep(int64 ns, String reason)
{
Timer t;
@@ -248,7 +248,7 @@
// No timers left - put goroutine to sleep.
timers.rescheduling = true;
g->isbackground = true;
- runtime·parkunlock(&timers.lock, "timer goroutine (idle)");
+ runtime·parkunlock(&timers.lock, runtime·gostringnocopy((byte*)"timer goroutine (idle)"));
g->isbackground = false;
continue;
}
