runtime: convert netpoll to Go
The common code is converted, epoll and kqueue are converted.
Windows and solaris are still C.
LGTM=rsc
R=golang-codereviews, rsc, dave
CC=golang-codereviews, iant, khr, rsc
https://golang.org/cl/132910043
diff --git a/src/pkg/runtime/asm_386.s b/src/pkg/runtime/asm_386.s
index 3c46b40..87dd1fa 100644
--- a/src/pkg/runtime/asm_386.s
+++ b/src/pkg/runtime/asm_386.s
@@ -598,6 +598,9 @@
MOVL AX, ret+8(FP)
RET
+TEXT runtime·xchguintptr(SB), NOSPLIT, $0-12
+ JMP runtime·xchg(SB)
+
TEXT runtime·procyield(SB),NOSPLIT,$0-0
MOVL cycles+0(FP), AX
again:
diff --git a/src/pkg/runtime/asm_amd64.s b/src/pkg/runtime/asm_amd64.s
index eb0795e..80c2ab3 100644
--- a/src/pkg/runtime/asm_amd64.s
+++ b/src/pkg/runtime/asm_amd64.s
@@ -709,6 +709,9 @@
MOVQ AX, ret+16(FP)
RET
+TEXT runtime·xchguintptr(SB), NOSPLIT, $0-24
+ JMP runtime·xchg64(SB)
+
TEXT runtime·procyield(SB),NOSPLIT,$0-0
MOVL cycles+0(FP), AX
again:
diff --git a/src/pkg/runtime/asm_amd64p32.s b/src/pkg/runtime/asm_amd64p32.s
index 512e923..a0e63b4 100644
--- a/src/pkg/runtime/asm_amd64p32.s
+++ b/src/pkg/runtime/asm_amd64p32.s
@@ -666,6 +666,16 @@
MOVQ AX, ret+16(FP)
RET
+TEXT runtime·xchgp(SB), NOSPLIT, $0-12
+ MOVL ptr+0(FP), BX
+ MOVL new+4(FP), AX
+ XCHGL AX, 0(BX)
+ MOVL AX, ret+8(FP)
+ RET
+
+TEXT runtime·xchguintptr(SB), NOSPLIT, $0-12
+ JMP runtime·xchg(SB)
+
TEXT runtime·procyield(SB),NOSPLIT,$0-0
MOVL cycles+0(FP), AX
again:
diff --git a/src/pkg/runtime/atomic_arm.c b/src/pkg/runtime/atomic_arm.c
index c15108e..5369820 100644
--- a/src/pkg/runtime/atomic_arm.c
+++ b/src/pkg/runtime/atomic_arm.c
@@ -55,6 +55,13 @@
}
#pragma textflag NOSPLIT
+void*
+runtime·xchguintptr(void* volatile* addr, void* v)
+{
+ return runtime·xchg((uint32*)addr, (uint32)v);
+}
+
+#pragma textflag NOSPLIT
void
runtime·procyield(uint32 cnt)
{
diff --git a/src/pkg/runtime/netpoll.go b/src/pkg/runtime/netpoll.go
new file mode 100644
index 0000000..5c410fc
--- /dev/null
+++ b/src/pkg/runtime/netpoll.go
@@ -0,0 +1,448 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows
+
+package runtime
+
+import "unsafe"
+
+// Integrated network poller (platform-independent part).
+// A particular implementation (epoll/kqueue) must define the following functions:
+// func netpollinit() // to initialize the poller
+// func netpollopen(fd uintptr, pd *pollDesc) int32 // to arm edge-triggered notifications
+// and associate fd with pd.
+// An implementation must call the following function to denote that the pd is ready.
+// func netpollready(rg, wg **gp, pd *pollDesc, mode int32)
+
+// pollDesc contains 2 binary semaphores, rg and wg, to park reader and writer
+// goroutines respectively. The semaphore can be in the following states:
+// pdReady - io readiness notification is pending;
+// a goroutine consumes the notification by changing the state to nil.
+// pdWait - a goroutine prepares to park on the semaphore, but not yet parked;
+// the goroutine commits to park by changing the state to G pointer,
+// or, alternatively, concurrent io notification changes the state to READY,
+// or, alternatively, concurrent timeout/close changes the state to nil.
+// G pointer - the goroutine is blocked on the semaphore;
+// io notification or timeout/close changes the state to READY or nil respectively
+// and unparks the goroutine.
+// nil - nothing of the above.
+const (
+ pdReady uintptr = 1
+ pdWait uintptr = 2
+)
+
+const pollBlockSize = 4 * 1024
+
+// Network poller descriptor.
+type pollDesc struct {
+ link *pollDesc // in pollcache, protected by pollcache.lock
+
+ // The lock protects pollOpen, pollSetDeadline, pollUnblock and deadlineimpl operations.
+ // This fully covers seq, rt and wt variables. fd is constant throughout the PollDesc lifetime.
+ // pollReset, pollWait, pollWaitCanceled and runtime·netpollready (IO readiness notification)
+ // proceed w/o taking the lock. So closing, rg, rd, wg and wd are manipulated
+ // in a lock-free way by all operations.
+ // NOTE(dvyukov): the following code uses uintptr to store *g (rg/wg),
+ // that will blow up when GC starts moving objects.
+ lock mutex // protectes the following fields
+ fd uintptr
+ closing bool
+ seq uintptr // protects from stale timers and ready notifications
+ rg uintptr // pdReady, pdWait, G waiting for read or nil
+ rt timer // read deadline timer (set if rt.f != nil)
+ rd int64 // read deadline
+ wg uintptr // pdReady, pdWait, G waiting for write or nil
+ wt timer // write deadline timer
+ wd int64 // write deadline
+ user unsafe.Pointer // user settable cookie
+}
+
+type pollCache struct {
+ lock mutex
+ first *pollDesc
+ // PollDesc objects must be type-stable,
+ // because we can get ready notification from epoll/kqueue
+ // after the descriptor is closed/reused.
+ // Stale notifications are detected using seq variable,
+ // seq is incremented when deadlines are changed or descriptor is reused.
+}
+
+var pollcache pollCache
+
+func netpollServerInit() {
+ netpollinit()
+}
+
+func netpollOpen(fd uintptr) (*pollDesc, int) {
+ pd := pollcache.alloc()
+ lock(&pd.lock)
+ if pd.wg != 0 && pd.wg != pdReady {
+ gothrow("netpollOpen: blocked write on free descriptor")
+ }
+ if pd.rg != 0 && pd.rg != pdReady {
+ gothrow("netpollOpen: blocked read on free descriptor")
+ }
+ pd.fd = fd
+ pd.closing = false
+ pd.seq++
+ pd.rg = 0
+ pd.rd = 0
+ pd.wg = 0
+ pd.wd = 0
+ unlock(&pd.lock)
+
+ errno := netpollopen(fd, pd)
+ return pd, int(errno)
+}
+
+func netpollClose(pd *pollDesc) {
+ if !pd.closing {
+ gothrow("runtime_pollClose: close w/o unblock")
+ }
+ if pd.wg != 0 && pd.wg != pdReady {
+ gothrow("netpollClose: blocked write on closing descriptor")
+ }
+ if pd.rg != 0 && pd.rg != pdReady {
+ gothrow("netpollClose: blocked read on closing descriptor")
+ }
+ netpollclose(uintptr(pd.fd))
+ pollcache.free(pd)
+}
+
+func (c *pollCache) free(pd *pollDesc) {
+ lock(&c.lock)
+ pd.link = c.first
+ c.first = pd
+ unlock(&c.lock)
+}
+
+func netpollReset(pd *pollDesc, mode int) int {
+ err := netpollcheckerr(pd, int32(mode))
+ if err != 0 {
+ return err
+ }
+ if mode == 'r' {
+ pd.rg = 0
+ } else if mode == 'w' {
+ pd.wg = 0
+ }
+ return 0
+}
+
+func netpollWait(pd *pollDesc, mode int) int {
+ err := netpollcheckerr(pd, int32(mode))
+ if err != 0 {
+ return err
+ }
+ // As for now only Solaris uses level-triggered IO.
+ if GOOS == "solaris" {
+ netpollarm(pd, mode)
+ }
+ for !netpollblock(pd, int32(mode), false) {
+ err = netpollcheckerr(pd, int32(mode))
+ if err != 0 {
+ return err
+ }
+ // Can happen if timeout has fired and unblocked us,
+ // but before we had a chance to run, timeout has been reset.
+ // Pretend it has not happened and retry.
+ }
+ return 0
+}
+
+func netpollWaitCanceled(pd *pollDesc, mode int) {
+ // This function is used only on windows after a failed attempt to cancel
+ // a pending async IO operation. Wait for ioready, ignore closing or timeouts.
+ for !netpollblock(pd, int32(mode), true) {
+ }
+}
+
+func netpollSetDeadline(pd *pollDesc, d int64, mode int) {
+ lock(&pd.lock)
+ if pd.closing {
+ unlock(&pd.lock)
+ return
+ }
+ pd.seq++ // invalidate current timers
+ // Reset current timers.
+ if pd.rt.f != nil {
+ deltimer(&pd.rt)
+ pd.rt.f = nil
+ }
+ if pd.wt.f != nil {
+ deltimer(&pd.wt)
+ pd.wt.f = nil
+ }
+ // Setup new timers.
+ if d != 0 && d <= nanotime() {
+ d = -1
+ }
+ if mode == 'r' || mode == 'r'+'w' {
+ pd.rd = d
+ }
+ if mode == 'w' || mode == 'r'+'w' {
+ pd.wd = d
+ }
+ if pd.rd > 0 && pd.rd == pd.wd {
+ pd.rt.f = netpollDeadline
+ pd.rt.when = pd.rd
+ // Copy current seq into the timer arg.
+ // Timer func will check the seq against current descriptor seq,
+ // if they differ the descriptor was reused or timers were reset.
+ pd.rt.arg = pd
+ pd.rt.seq = pd.seq
+ addtimer(&pd.rt)
+ } else {
+ if pd.rd > 0 {
+ pd.rt.f = netpollReadDeadline
+ pd.rt.when = pd.rd
+ pd.rt.arg = pd
+ pd.rt.seq = pd.seq
+ addtimer(&pd.rt)
+ }
+ if pd.wd > 0 {
+ pd.wt.f = netpollWriteDeadline
+ pd.wt.when = pd.wd
+ pd.wt.arg = pd
+ pd.wt.seq = pd.seq
+ addtimer(&pd.wt)
+ }
+ }
+ // If we set the new deadline in the past, unblock currently pending IO if any.
+ var rg, wg *g
+ atomicstorep(unsafe.Pointer(&wg), nil) // full memory barrier between stores to rd/wd and load of rg/wg in netpollunblock
+ if pd.rd < 0 {
+ rg = netpollunblock(pd, 'r', false)
+ }
+ if pd.wd < 0 {
+ wg = netpollunblock(pd, 'w', false)
+ }
+ unlock(&pd.lock)
+ if rg != nil {
+ goready(rg)
+ }
+ if wg != nil {
+ goready(wg)
+ }
+}
+
+func netpollUnblock(pd *pollDesc) {
+ lock(&pd.lock)
+ if pd.closing {
+ gothrow("netpollUnblock: already closing")
+ }
+ pd.closing = true
+ pd.seq++
+ var rg, wg *g
+ atomicstorep(unsafe.Pointer(&rg), nil) // full memory barrier between store to closing and read of rg/wg in netpollunblock
+ rg = netpollunblock(pd, 'r', false)
+ wg = netpollunblock(pd, 'w', false)
+ if pd.rt.f != nil {
+ deltimer(&pd.rt)
+ pd.rt.f = nil
+ }
+ if pd.wt.f != nil {
+ deltimer(&pd.wt)
+ pd.wt.f = nil
+ }
+ unlock(&pd.lock)
+ if rg != nil {
+ goready(rg)
+ }
+ if wg != nil {
+ goready(wg)
+ }
+}
+
+func netpollfd(pd *pollDesc) uintptr {
+ return pd.fd
+}
+
+func netpolluser(pd *pollDesc) *unsafe.Pointer {
+ return &pd.user
+}
+
+func netpollclosing(pd *pollDesc) bool {
+ return pd.closing
+}
+
+func netpolllock(pd *pollDesc) {
+ lock(&pd.lock)
+}
+
+func netpollunlock(pd *pollDesc) {
+ lock(&pd.lock)
+}
+
+// make pd ready, newly runnable goroutines (if any) are returned in rg/wg
+func netpollready(gpp **g, pd *pollDesc, mode int32) {
+ var rg, wg *g
+ if mode == 'r' || mode == 'r'+'w' {
+ rg = netpollunblock(pd, 'r', true)
+ }
+ if mode == 'w' || mode == 'r'+'w' {
+ wg = netpollunblock(pd, 'w', true)
+ }
+ if rg != nil {
+ rg.schedlink = *gpp
+ *gpp = rg
+ }
+ if wg != nil {
+ wg.schedlink = *gpp
+ *gpp = wg
+ }
+}
+
+func netpollcheckerr(pd *pollDesc, mode int32) int {
+ if pd.closing {
+ return 1 // errClosing
+ }
+ if (mode == 'r' && pd.rd < 0) || (mode == 'w' && pd.wd < 0) {
+ return 2 // errTimeout
+ }
+ return 0
+}
+
+func netpollblockcommit(gp *g, gpp unsafe.Pointer) bool {
+ return casuintptr((*uintptr)(gpp), pdWait, uintptr(unsafe.Pointer(gp)))
+}
+
+// returns true if IO is ready, or false if timedout or closed
+// waitio - wait only for completed IO, ignore errors
+func netpollblock(pd *pollDesc, mode int32, waitio bool) bool {
+ gpp := &pd.rg
+ if mode == 'w' {
+ gpp = &pd.wg
+ }
+
+ // set the gpp semaphore to WAIT
+ for {
+ old := *gpp
+ if old == pdReady {
+ *gpp = 0
+ return true
+ }
+ if old != 0 {
+ gothrow("netpollblock: double wait")
+ }
+ if casuintptr(gpp, 0, pdWait) {
+ break
+ }
+ }
+
+ // need to recheck error states after setting gpp to WAIT
+ // this is necessary because runtime_pollUnblock/runtime_pollSetDeadline/deadlineimpl
+ // do the opposite: store to closing/rd/wd, membarrier, load of rg/wg
+ if waitio || netpollcheckerr(pd, mode) == 0 {
+ f := netpollblockcommit
+ gopark(**(**unsafe.Pointer)(unsafe.Pointer(&f)), unsafe.Pointer(gpp), "IO wait")
+ }
+ // be careful to not lose concurrent READY notification
+ old := xchguintptr(gpp, 0)
+ if old > pdWait {
+ gothrow("netpollblock: corrupted state")
+ }
+ return old == pdReady
+}
+
+func netpollunblock(pd *pollDesc, mode int32, ioready bool) *g {
+ gpp := &pd.rg
+ if mode == 'w' {
+ gpp = &pd.wg
+ }
+
+ for {
+ old := *gpp
+ if old == pdReady {
+ return nil
+ }
+ if old == 0 && !ioready {
+ // Only set READY for ioready. runtime_pollWait
+ // will check for timeout/cancel before waiting.
+ return nil
+ }
+ var new uintptr
+ if ioready {
+ new = pdReady
+ }
+ if casuintptr(gpp, old, new) {
+ if old == pdReady || old == pdWait {
+ old = 0
+ }
+ return (*g)(unsafe.Pointer(old))
+ }
+ }
+}
+
+func netpolldeadlineimpl(pd *pollDesc, seq uintptr, read, write bool) {
+ lock(&pd.lock)
+ // Seq arg is seq when the timer was set.
+ // If it's stale, ignore the timer event.
+ if seq != pd.seq {
+ // The descriptor was reused or timers were reset.
+ unlock(&pd.lock)
+ return
+ }
+ var rg *g
+ if read {
+ if pd.rd <= 0 || pd.rt.f == nil {
+ gothrow("netpollDeadlineImpl: inconsistent read deadline")
+ }
+ pd.rd = -1
+ atomicstorep(unsafe.Pointer(&pd.rt.f), nil) // full memory barrier between store to rd and load of rg in netpollunblock
+ rg = netpollunblock(pd, 'r', false)
+ }
+ var wg *g
+ if write {
+ if pd.wd <= 0 || pd.wt.f == nil && !read {
+ gothrow("netpolldeadlineimpl: inconsistent write deadline")
+ }
+ pd.wd = -1
+ atomicstorep(unsafe.Pointer(&pd.wt.f), nil) // full memory barrier between store to wd and load of wg in netpollunblock
+ wg = netpollunblock(pd, 'w', false)
+ }
+ unlock(&pd.lock)
+ if rg != nil {
+ goready(rg)
+ }
+ if wg != nil {
+ goready(wg)
+ }
+}
+
+func netpollDeadline(arg interface{}, seq uintptr) {
+ netpolldeadlineimpl(arg.(*pollDesc), seq, true, true)
+}
+
+func netpollReadDeadline(arg interface{}, seq uintptr) {
+ netpolldeadlineimpl(arg.(*pollDesc), seq, true, false)
+}
+
+func netpollWriteDeadline(arg interface{}, seq uintptr) {
+ netpolldeadlineimpl(arg.(*pollDesc), seq, false, true)
+}
+
+func (c *pollCache) alloc() *pollDesc {
+ lock(&c.lock)
+ if c.first == nil {
+ const pdSize = unsafe.Sizeof(pollDesc{})
+ n := pollBlockSize / pdSize
+ if n == 0 {
+ n = 1
+ }
+ // Must be in non-GC memory because can be referenced
+ // only from epoll/kqueue internals.
+ mem := persistentalloc(n*pdSize, 0, &memstats.other_sys)
+ for i := uintptr(0); i < n; i++ {
+ pd := (*pollDesc)(add(mem, i*pdSize))
+ pd.link = c.first
+ c.first = pd
+ }
+ }
+ pd := c.first
+ c.first = pd.link
+ unlock(&c.lock)
+ return pd
+}
diff --git a/src/pkg/runtime/netpoll.goc b/src/pkg/runtime/netpoll.goc
deleted file mode 100644
index 446e78e..0000000
--- a/src/pkg/runtime/netpoll.goc
+++ /dev/null
@@ -1,481 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows
-
-package net
-
-#include "runtime.h"
-#include "defs_GOOS_GOARCH.h"
-#include "arch_GOARCH.h"
-#include "malloc.h"
-
-// Integrated network poller (platform-independent part).
-// A particular implementation (epoll/kqueue) must define the following functions:
-// void runtime·netpollinit(void); // to initialize the poller
-// int32 runtime·netpollopen(uintptr fd, PollDesc *pd); // to arm edge-triggered notifications
- // and associate fd with pd.
-// An implementation must call the following function to denote that the pd is ready.
-// void runtime·netpollready(G **gpp, PollDesc *pd, int32 mode);
-
-// PollDesc contains 2 binary semaphores, rg and wg, to park reader and writer
-// goroutines respectively. The semaphore can be in the following states:
-// READY - io readiness notification is pending;
-// a goroutine consumes the notification by changing the state to nil.
-// WAIT - a goroutine prepares to park on the semaphore, but not yet parked;
-// the goroutine commits to park by changing the state to G pointer,
-// or, alternatively, concurrent io notification changes the state to READY,
-// or, alternatively, concurrent timeout/close changes the state to nil.
-// G pointer - the goroutine is blocked on the semaphore;
-// io notification or timeout/close changes the state to READY or nil respectively
-// and unparks the goroutine.
-// nil - nothing of the above.
-#define READY ((G*)1)
-#define WAIT ((G*)2)
-
-enum
-{
- PollBlockSize = 4*1024,
-};
-
-// time.go defines the layout of this structure.
-// Keep in sync with time.go.
-typedef struct Timer Timer;
-struct Timer
-{
- intgo i;
- int64 when;
- int64 period;
- FuncVal *fv;
- Eface arg;
-};
-
-void runtime·addtimer(Timer*);
-void runtime·deltimer(Timer*);
-
-struct PollDesc
-{
- PollDesc* link; // in pollcache, protected by pollcache.lock
-
- // The lock protects pollOpen, pollSetDeadline, pollUnblock and deadlineimpl operations.
- // This fully covers seq, rt and wt variables. fd is constant throughout the PollDesc lifetime.
- // pollReset, pollWait, pollWaitCanceled and runtime·netpollready (IO rediness notification)
- // proceed w/o taking the lock. So closing, rg, rd, wg and wd are manipulated
- // in a lock-free way by all operations.
- Mutex lock; // protects the following fields
- uintptr fd;
- bool closing;
- uintptr seq; // protects from stale timers and ready notifications
- G* rg; // READY, WAIT, G waiting for read or nil
- Timer rt; // read deadline timer (set if rt.fv != nil)
- int64 rd; // read deadline
- G* wg; // READY, WAIT, G waiting for write or nil
- Timer wt; // write deadline timer
- int64 wd; // write deadline
- void* user; // user settable cookie
-};
-
-static struct
-{
- Mutex lock;
- PollDesc* first;
- // PollDesc objects must be type-stable,
- // because we can get ready notification from epoll/kqueue
- // after the descriptor is closed/reused.
- // Stale notifications are detected using seq variable,
- // seq is incremented when deadlines are changed or descriptor is reused.
-} pollcache;
-
-static bool netpollblock(PollDesc*, int32, bool);
-static G* netpollunblock(PollDesc*, int32, bool);
-static void deadline(Eface);
-static void readDeadline(Eface);
-static void writeDeadline(Eface);
-static PollDesc* allocPollDesc(void);
-static intgo checkerr(PollDesc *pd, int32 mode);
-
-static FuncVal deadlineFn = {(void(*)(void))deadline};
-static FuncVal readDeadlineFn = {(void(*)(void))readDeadline};
-static FuncVal writeDeadlineFn = {(void(*)(void))writeDeadline};
-
-// runtimeNano returns the current value of the runtime clock in nanoseconds.
-func runtimeNano() (ns int64) {
- ns = runtime·nanotime();
-}
-
-func runtime_pollServerInit() {
- runtime·netpollinit();
-}
-
-func runtime_pollOpen(fd uintptr) (pd *PollDesc, errno int) {
- pd = allocPollDesc();
- runtime·lock(&pd->lock);
- if(pd->wg != nil && pd->wg != READY)
- runtime·throw("runtime_pollOpen: blocked write on free descriptor");
- if(pd->rg != nil && pd->rg != READY)
- runtime·throw("runtime_pollOpen: blocked read on free descriptor");
- pd->fd = fd;
- pd->closing = false;
- pd->seq++;
- pd->rg = nil;
- pd->rd = 0;
- pd->wg = nil;
- pd->wd = 0;
- runtime·unlock(&pd->lock);
-
- errno = runtime·netpollopen(fd, pd);
-}
-
-func runtime_pollClose(pd *PollDesc) {
- if(!pd->closing)
- runtime·throw("runtime_pollClose: close w/o unblock");
- if(pd->wg != nil && pd->wg != READY)
- runtime·throw("runtime_pollClose: blocked write on closing descriptor");
- if(pd->rg != nil && pd->rg != READY)
- runtime·throw("runtime_pollClose: blocked read on closing descriptor");
- runtime·netpollclose(pd->fd);
- runtime·lock(&pollcache.lock);
- pd->link = pollcache.first;
- pollcache.first = pd;
- runtime·unlock(&pollcache.lock);
-}
-
-func runtime_pollReset(pd *PollDesc, mode int) (err int) {
- err = checkerr(pd, mode);
- if(err)
- goto ret;
- if(mode == 'r')
- pd->rg = nil;
- else if(mode == 'w')
- pd->wg = nil;
-ret:
-}
-
-func runtime_pollWait(pd *PollDesc, mode int) (err int) {
- err = checkerr(pd, mode);
- if(err == 0) {
- // As for now only Solaris uses level-triggered IO.
- if(Solaris)
- runtime·netpollarm(pd, mode);
- while(!netpollblock(pd, mode, false)) {
- err = checkerr(pd, mode);
- if(err != 0)
- break;
- // Can happen if timeout has fired and unblocked us,
- // but before we had a chance to run, timeout has been reset.
- // Pretend it has not happened and retry.
- }
- }
-}
-
-func runtime_pollWaitCanceled(pd *PollDesc, mode int) {
- // This function is used only on windows after a failed attempt to cancel
- // a pending async IO operation. Wait for ioready, ignore closing or timeouts.
- while(!netpollblock(pd, mode, true))
- ;
-}
-
-func runtime_pollSetDeadline(pd *PollDesc, d int64, mode int) {
- G *rg, *wg;
-
- runtime·lock(&pd->lock);
- if(pd->closing) {
- runtime·unlock(&pd->lock);
- return;
- }
- pd->seq++; // invalidate current timers
- // Reset current timers.
- if(pd->rt.fv) {
- runtime·deltimer(&pd->rt);
- pd->rt.fv = nil;
- }
- if(pd->wt.fv) {
- runtime·deltimer(&pd->wt);
- pd->wt.fv = nil;
- }
- // Setup new timers.
- if(d != 0 && d <= runtime·nanotime())
- d = -1;
- if(mode == 'r' || mode == 'r'+'w')
- pd->rd = d;
- if(mode == 'w' || mode == 'r'+'w')
- pd->wd = d;
- if(pd->rd > 0 && pd->rd == pd->wd) {
- pd->rt.fv = &deadlineFn;
- pd->rt.when = pd->rd;
- // Copy current seq into the timer arg.
- // Timer func will check the seq against current descriptor seq,
- // if they differ the descriptor was reused or timers were reset.
- pd->rt.arg.type = (Type*)pd->seq;
- pd->rt.arg.data = pd;
- runtime·addtimer(&pd->rt);
- } else {
- if(pd->rd > 0) {
- pd->rt.fv = &readDeadlineFn;
- pd->rt.when = pd->rd;
- pd->rt.arg.type = (Type*)pd->seq;
- pd->rt.arg.data = pd;
- runtime·addtimer(&pd->rt);
- }
- if(pd->wd > 0) {
- pd->wt.fv = &writeDeadlineFn;
- pd->wt.when = pd->wd;
- pd->wt.arg.type = (Type*)pd->seq;
- pd->wt.arg.data = pd;
- runtime·addtimer(&pd->wt);
- }
- }
- // If we set the new deadline in the past, unblock currently pending IO if any.
- rg = nil;
- runtime·atomicstorep(&wg, nil); // full memory barrier between stores to rd/wd and load of rg/wg in netpollunblock
- if(pd->rd < 0)
- rg = netpollunblock(pd, 'r', false);
- if(pd->wd < 0)
- wg = netpollunblock(pd, 'w', false);
- runtime·unlock(&pd->lock);
- if(rg)
- runtime·ready(rg);
- if(wg)
- runtime·ready(wg);
-}
-
-func runtime_pollUnblock(pd *PollDesc) {
- G *rg, *wg;
-
- runtime·lock(&pd->lock);
- if(pd->closing)
- runtime·throw("runtime_pollUnblock: already closing");
- pd->closing = true;
- pd->seq++;
- runtime·atomicstorep(&rg, nil); // full memory barrier between store to closing and read of rg/wg in netpollunblock
- rg = netpollunblock(pd, 'r', false);
- wg = netpollunblock(pd, 'w', false);
- if(pd->rt.fv) {
- runtime·deltimer(&pd->rt);
- pd->rt.fv = nil;
- }
- if(pd->wt.fv) {
- runtime·deltimer(&pd->wt);
- pd->wt.fv = nil;
- }
- runtime·unlock(&pd->lock);
- if(rg)
- runtime·ready(rg);
- if(wg)
- runtime·ready(wg);
-}
-
-uintptr
-runtime·netpollfd(PollDesc *pd)
-{
- return pd->fd;
-}
-
-void**
-runtime·netpolluser(PollDesc *pd)
-{
- return &pd->user;
-}
-
-bool
-runtime·netpollclosing(PollDesc *pd)
-{
- return pd->closing;
-}
-
-void
-runtime·netpolllock(PollDesc *pd)
-{
- runtime·lock(&pd->lock);
-}
-
-void
-runtime·netpollunlock(PollDesc *pd)
-{
- runtime·unlock(&pd->lock);
-}
-
-// make pd ready, newly runnable goroutines (if any) are enqueued info gpp list
-void
-runtime·netpollready(G **gpp, PollDesc *pd, int32 mode)
-{
- G *rg, *wg;
-
- rg = wg = nil;
- if(mode == 'r' || mode == 'r'+'w')
- rg = netpollunblock(pd, 'r', true);
- if(mode == 'w' || mode == 'r'+'w')
- wg = netpollunblock(pd, 'w', true);
- if(rg) {
- rg->schedlink = *gpp;
- *gpp = rg;
- }
- if(wg) {
- wg->schedlink = *gpp;
- *gpp = wg;
- }
-}
-
-static intgo
-checkerr(PollDesc *pd, int32 mode)
-{
- if(pd->closing)
- return 1; // errClosing
- if((mode == 'r' && pd->rd < 0) || (mode == 'w' && pd->wd < 0))
- return 2; // errTimeout
- return 0;
-}
-
-static bool
-blockcommit(G *gp, G **gpp)
-{
- return runtime·casp(gpp, WAIT, gp);
-}
-
-// returns true if IO is ready, or false if timedout or closed
-// waitio - wait only for completed IO, ignore errors
-static bool
-netpollblock(PollDesc *pd, int32 mode, bool waitio)
-{
- G **gpp, *old;
-
- gpp = &pd->rg;
- if(mode == 'w')
- gpp = &pd->wg;
-
- // set the gpp semaphore to WAIT
- for(;;) {
- old = *gpp;
- if(old == READY) {
- *gpp = nil;
- return true;
- }
- if(old != nil)
- runtime·throw("netpollblock: double wait");
- if(runtime·casp(gpp, nil, WAIT))
- break;
- }
-
- // need to recheck error states after setting gpp to WAIT
- // this is necessary because runtime_pollUnblock/runtime_pollSetDeadline/deadlineimpl
- // do the opposite: store to closing/rd/wd, membarrier, load of rg/wg
- if(waitio || checkerr(pd, mode) == 0)
- runtime·park((bool(*)(G*, void*))blockcommit, gpp, runtime·gostringnocopy((byte*)"IO wait"));
- // be careful to not lose concurrent READY notification
- old = runtime·xchgp(gpp, nil);
- if(old > WAIT)
- runtime·throw("netpollblock: corrupted state");
- return old == READY;
-}
-
-static G*
-netpollunblock(PollDesc *pd, int32 mode, bool ioready)
-{
- G **gpp, *old, *new;
-
- gpp = &pd->rg;
- if(mode == 'w')
- gpp = &pd->wg;
-
- for(;;) {
- old = *gpp;
- if(old == READY)
- return nil;
- if(old == nil && !ioready) {
- // Only set READY for ioready. runtime_pollWait
- // will check for timeout/cancel before waiting.
- return nil;
- }
- new = nil;
- if(ioready)
- new = READY;
- if(runtime·casp(gpp, old, new))
- break;
- }
- if(old > WAIT)
- return old; // must be G*
- return nil;
-}
-
-static void
-deadlineimpl(Eface arg, bool read, bool write)
-{
- PollDesc *pd;
- uint32 seq;
- G *rg, *wg;
-
- pd = (PollDesc*)arg.data;
- // This is the seq when the timer was set.
- // If it's stale, ignore the timer event.
- seq = (uintptr)arg.type;
- rg = wg = nil;
- runtime·lock(&pd->lock);
- if(seq != pd->seq) {
- // The descriptor was reused or timers were reset.
- runtime·unlock(&pd->lock);
- return;
- }
- if(read) {
- if(pd->rd <= 0 || pd->rt.fv == nil)
- runtime·throw("deadlineimpl: inconsistent read deadline");
- pd->rd = -1;
- runtime·atomicstorep(&pd->rt.fv, nil); // full memory barrier between store to rd and load of rg in netpollunblock
- rg = netpollunblock(pd, 'r', false);
- }
- if(write) {
- if(pd->wd <= 0 || (pd->wt.fv == nil && !read))
- runtime·throw("deadlineimpl: inconsistent write deadline");
- pd->wd = -1;
- runtime·atomicstorep(&pd->wt.fv, nil); // full memory barrier between store to wd and load of wg in netpollunblock
- wg = netpollunblock(pd, 'w', false);
- }
- runtime·unlock(&pd->lock);
- if(rg)
- runtime·ready(rg);
- if(wg)
- runtime·ready(wg);
-}
-
-static void
-deadline(Eface arg)
-{
- deadlineimpl(arg, true, true);
-}
-
-static void
-readDeadline(Eface arg)
-{
- deadlineimpl(arg, true, false);
-}
-
-static void
-writeDeadline(Eface arg)
-{
- deadlineimpl(arg, false, true);
-}
-
-static PollDesc*
-allocPollDesc(void)
-{
- PollDesc *pd;
- uint32 i, n;
-
- runtime·lock(&pollcache.lock);
- if(pollcache.first == nil) {
- n = PollBlockSize/sizeof(*pd);
- if(n == 0)
- n = 1;
- // Must be in non-GC memory because can be referenced
- // only from epoll/kqueue internals.
- pd = runtime·persistentalloc(n*sizeof(*pd), 0, &mstats.other_sys);
- for(i = 0; i < n; i++) {
- pd[i].link = pollcache.first;
- pollcache.first = &pd[i];
- }
- }
- pd = pollcache.first;
- pollcache.first = pd->link;
- runtime·unlock(&pollcache.lock);
- return pd;
-}
diff --git a/src/pkg/runtime/netpoll_epoll.c b/src/pkg/runtime/netpoll_epoll.c
deleted file mode 100644
index 9d6c205..0000000
--- a/src/pkg/runtime/netpoll_epoll.c
+++ /dev/null
@@ -1,103 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build linux
-
-#include "runtime.h"
-#include "defs_GOOS_GOARCH.h"
-
-int32 runtime·epollcreate(int32 size);
-int32 runtime·epollcreate1(int32 flags);
-int32 runtime·epollctl(int32 epfd, int32 op, int32 fd, EpollEvent *ev);
-int32 runtime·epollwait(int32 epfd, EpollEvent *ev, int32 nev, int32 timeout);
-void runtime·closeonexec(int32 fd);
-
-static int32 epfd = -1; // epoll descriptor
-
-void
-runtime·netpollinit(void)
-{
- epfd = runtime·epollcreate1(EPOLL_CLOEXEC);
- if(epfd >= 0)
- return;
- epfd = runtime·epollcreate(1024);
- if(epfd >= 0) {
- runtime·closeonexec(epfd);
- return;
- }
- runtime·printf("netpollinit: failed to create descriptor (%d)\n", -epfd);
- runtime·throw("netpollinit: failed to create descriptor");
-}
-
-int32
-runtime·netpollopen(uintptr fd, PollDesc *pd)
-{
- EpollEvent ev;
- int32 res;
-
- ev.events = EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET;
- *(uint64*)ev.data = (uint64)(uintptr)pd;
- res = runtime·epollctl(epfd, EPOLL_CTL_ADD, (int32)fd, &ev);
- return -res;
-}
-
-int32
-runtime·netpollclose(uintptr fd)
-{
- EpollEvent ev;
- int32 res;
-
- res = runtime·epollctl(epfd, EPOLL_CTL_DEL, (int32)fd, &ev);
- return -res;
-}
-
-void
-runtime·netpollarm(PollDesc* pd, int32 mode)
-{
- USED(pd);
- USED(mode);
- runtime·throw("unused");
-}
-
-// polls for ready network connections
-// returns list of goroutines that become runnable
-G*
-runtime·netpoll(bool block)
-{
- static int32 lasterr;
- EpollEvent events[128], *ev;
- int32 n, i, waitms, mode;
- G *gp;
-
- if(epfd == -1)
- return nil;
- waitms = -1;
- if(!block)
- waitms = 0;
-retry:
- n = runtime·epollwait(epfd, events, nelem(events), waitms);
- if(n < 0) {
- if(n != -EINTR && n != lasterr) {
- lasterr = n;
- runtime·printf("runtime: epollwait on fd %d failed with %d\n", epfd, -n);
- }
- goto retry;
- }
- gp = nil;
- for(i = 0; i < n; i++) {
- ev = &events[i];
- if(ev->events == 0)
- continue;
- mode = 0;
- if(ev->events & (EPOLLIN|EPOLLRDHUP|EPOLLHUP|EPOLLERR))
- mode += 'r';
- if(ev->events & (EPOLLOUT|EPOLLHUP|EPOLLERR))
- mode += 'w';
- if(mode)
- runtime·netpollready(&gp, (void*)(uintptr)*(uint64*)ev->data, mode);
- }
- if(block && gp == nil)
- goto retry;
- return gp;
-}
diff --git a/src/pkg/runtime/netpoll_epoll.go b/src/pkg/runtime/netpoll_epoll.go
new file mode 100644
index 0000000..ecfc9cd
--- /dev/null
+++ b/src/pkg/runtime/netpoll_epoll.go
@@ -0,0 +1,97 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+
+package runtime
+
+import "unsafe"
+
+func epollcreate(size int32) int32
+func epollcreate1(flags int32) int32
+
+//go:noescape
+func epollctl(epfd, op, fd int32, ev *epollevent) int32
+
+//go:noescape
+func epollwait(epfd int32, ev *epollevent, nev, timeout int32) int32
+func closeonexec(fd int32)
+
+var (
+ epfd int32 = -1 // epoll descriptor
+ netpolllasterr int32
+)
+
+func netpollinit() {
+ epfd = epollcreate1(_EPOLL_CLOEXEC)
+ if epfd >= 0 {
+ return
+ }
+ epfd = epollcreate(1024)
+ if epfd >= 0 {
+ closeonexec(epfd)
+ return
+ }
+ println("netpollinit: failed to create epoll descriptor", -epfd)
+ gothrow("netpollinit: failed to create descriptor")
+}
+
+func netpollopen(fd uintptr, pd *pollDesc) int32 {
+ var ev epollevent
+ ev.events = _EPOLLIN | _EPOLLOUT | _EPOLLRDHUP | _EPOLLET
+ *(**pollDesc)(unsafe.Pointer(&ev.data)) = pd
+ return -epollctl(epfd, _EPOLL_CTL_ADD, int32(fd), &ev)
+}
+
+func netpollclose(fd uintptr) int32 {
+ var ev epollevent
+ return -epollctl(epfd, _EPOLL_CTL_DEL, int32(fd), &ev)
+}
+
+func netpollarm(pd *pollDesc, mode int) {
+ gothrow("unused")
+}
+
+// polls for ready network connections
+// returns list of goroutines that become runnable
+func netpoll(block bool) (gp *g) {
+ if epfd == -1 {
+ return
+ }
+ waitms := int32(-1)
+ if !block {
+ waitms = 0
+ }
+ var events [128]epollevent
+retry:
+ n := epollwait(epfd, &events[0], int32(len(events)), waitms)
+ if n < 0 {
+ if n != -_EINTR && n != netpolllasterr {
+ netpolllasterr = n
+ println("runtime: epollwait on fd", epfd, "failed with", -n)
+ }
+ goto retry
+ }
+ for i := int32(0); i < n; i++ {
+ ev := &events[i]
+ if ev.events == 0 {
+ continue
+ }
+ var mode int32
+ if ev.events&(_EPOLLIN|_EPOLLRDHUP|_EPOLLHUP|_EPOLLERR) != 0 {
+ mode += 'r'
+ }
+ if ev.events&(_EPOLLOUT|_EPOLLHUP|_EPOLLERR) != 0 {
+ mode += 'w'
+ }
+ if mode != 0 {
+ pd := *(**pollDesc)(unsafe.Pointer(&ev.data))
+ netpollready((**g)(noescape(unsafe.Pointer(&gp))), pd, mode)
+ }
+ }
+ if block && gp == nil {
+ goto retry
+ }
+ return gp
+}
diff --git a/src/pkg/runtime/netpoll_kqueue.c b/src/pkg/runtime/netpoll_kqueue.c
deleted file mode 100644
index 6b7fc86..0000000
--- a/src/pkg/runtime/netpoll_kqueue.c
+++ /dev/null
@@ -1,111 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build darwin dragonfly freebsd netbsd openbsd
-
-#include "runtime.h"
-#include "defs_GOOS_GOARCH.h"
-#include "os_GOOS.h"
-
-// Integrated network poller (kqueue-based implementation).
-
-int32 runtime·kqueue(void);
-int32 runtime·kevent(int32, KeventT*, int32, KeventT*, int32, Timespec*);
-void runtime·closeonexec(int32);
-
-static int32 kq = -1;
-
-void
-runtime·netpollinit(void)
-{
- kq = runtime·kqueue();
- if(kq < 0) {
- runtime·printf("netpollinit: kqueue failed with %d\n", -kq);
- runtime·throw("netpollinit: kqueue failed");
- }
- runtime·closeonexec(kq);
-}
-
-int32
-runtime·netpollopen(uintptr fd, PollDesc *pd)
-{
- KeventT ev[2];
- int32 n;
-
- // Arm both EVFILT_READ and EVFILT_WRITE in edge-triggered mode (EV_CLEAR)
- // for the whole fd lifetime. The notifications are automatically unregistered
- // when fd is closed.
- ev[0].ident = (uint32)fd;
- ev[0].filter = EVFILT_READ;
- ev[0].flags = EV_ADD|EV_CLEAR;
- ev[0].fflags = 0;
- ev[0].data = 0;
- ev[0].udata = (kevent_udata)pd;
- ev[1] = ev[0];
- ev[1].filter = EVFILT_WRITE;
- n = runtime·kevent(kq, ev, 2, nil, 0, nil);
- if(n < 0)
- return -n;
- return 0;
-}
-
-int32
-runtime·netpollclose(uintptr fd)
-{
- // Don't need to unregister because calling close()
- // on fd will remove any kevents that reference the descriptor.
- USED(fd);
- return 0;
-}
-
-void
-runtime·netpollarm(PollDesc* pd, int32 mode)
-{
- USED(pd, mode);
- runtime·throw("unused");
-}
-
-// Polls for ready network connections.
-// Returns list of goroutines that become runnable.
-G*
-runtime·netpoll(bool block)
-{
- static int32 lasterr;
- KeventT events[64], *ev;
- Timespec ts, *tp;
- int32 n, i, mode;
- G *gp;
-
- if(kq == -1)
- return nil;
- tp = nil;
- if(!block) {
- ts.tv_sec = 0;
- ts.tv_nsec = 0;
- tp = &ts;
- }
- gp = nil;
-retry:
- n = runtime·kevent(kq, nil, 0, events, nelem(events), tp);
- if(n < 0) {
- if(n != -EINTR && n != lasterr) {
- lasterr = n;
- runtime·printf("runtime: kevent on fd %d failed with %d\n", kq, -n);
- }
- goto retry;
- }
- for(i = 0; i < n; i++) {
- ev = &events[i];
- mode = 0;
- if(ev->filter == EVFILT_READ)
- mode += 'r';
- if(ev->filter == EVFILT_WRITE)
- mode += 'w';
- if(mode)
- runtime·netpollready(&gp, (PollDesc*)ev->udata, mode);
- }
- if(block && gp == nil)
- goto retry;
- return gp;
-}
diff --git a/src/pkg/runtime/netpoll_kqueue.go b/src/pkg/runtime/netpoll_kqueue.go
new file mode 100644
index 0000000..d6d55b9
--- /dev/null
+++ b/src/pkg/runtime/netpoll_kqueue.go
@@ -0,0 +1,101 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd netbsd openbsd
+
+package runtime
+
+// Integrated network poller (kqueue-based implementation).
+
+import "unsafe"
+
+func kqueue() int32
+
+//go:noescape
+func kevent(kq int32, ch *keventt, nch int32, ev *keventt, nev int32, ts *timespec) int32
+func closeonexec(fd int32)
+
+var (
+ kq int32 = -1
+ netpolllasterr int32
+)
+
+func netpollinit() {
+ kq = kqueue()
+ if kq < 0 {
+ println("netpollinit: kqueue failed with", -kq)
+ gothrow("netpollinit: kqueue failed")
+ }
+ closeonexec(kq)
+}
+
+func netpollopen(fd uintptr, pd *pollDesc) int32 {
+ // Arm both EVFILT_READ and EVFILT_WRITE in edge-triggered mode (EV_CLEAR)
+ // for the whole fd lifetime. The notifications are automatically unregistered
+ // when fd is closed.
+ var ev [2]keventt
+ *(*uintptr)(unsafe.Pointer(&ev[0].ident)) = fd
+ ev[0].filter = _EVFILT_READ
+ ev[0].flags = _EV_ADD | _EV_CLEAR
+ ev[0].fflags = 0
+ ev[0].data = 0
+ ev[0].udata = (*byte)(unsafe.Pointer(pd))
+ ev[1] = ev[0]
+ ev[1].filter = _EVFILT_WRITE
+ n := kevent(kq, &ev[0], 2, nil, 0, nil)
+ if n < 0 {
+ return -n
+ }
+ return 0
+}
+
+func netpollclose(fd uintptr) int32 {
+ // Don't need to unregister because calling close()
+ // on fd will remove any kevents that reference the descriptor.
+ return 0
+}
+
+func netpollarm(pd *pollDesc, mode int) {
+ gothrow("unused")
+}
+
+// Polls for ready network connections.
+// Returns list of goroutines that become runnable.
+func netpoll(block bool) (gp *g) {
+ if kq == -1 {
+ return
+ }
+ var tp *timespec
+ var ts timespec
+ if !block {
+ tp = &ts
+ }
+ var events [64]keventt
+retry:
+ n := kevent(kq, nil, 0, &events[0], int32(len(events)), tp)
+ if n < 0 {
+ if n != -_EINTR && n != netpolllasterr {
+ netpolllasterr = n
+ println("runtime: kevent on fd", kq, "failed with", -n)
+ }
+ goto retry
+ }
+ for i := 0; i < int(n); i++ {
+ ev := &events[i]
+ var mode int32
+ if ev.filter == _EVFILT_READ {
+ mode += 'r'
+ }
+ if ev.filter == _EVFILT_WRITE {
+ mode += 'w'
+ }
+ if mode != 0 {
+ netpollready((**g)(noescape(unsafe.Pointer(&gp))), (*pollDesc)(unsafe.Pointer(ev.udata)), mode)
+ }
+ }
+ if block && gp == nil {
+ goto retry
+ }
+ return gp
+}
diff --git a/src/pkg/runtime/netpoll_nacl.c b/src/pkg/runtime/netpoll_nacl.c
deleted file mode 100644
index b75753a..0000000
--- a/src/pkg/runtime/netpoll_nacl.c
+++ /dev/null
@@ -1,37 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-#include "runtime.h"
-#include "defs_GOOS_GOARCH.h"
-#include "os_GOOS.h"
-
-// Fake network poller for NaCl.
-// Should never be used, because NaCl network connections do not honor "SetNonblock".
-
-void
-runtime·netpollinit(void)
-{
-}
-
-int32
-runtime·netpollopen(uintptr fd, PollDesc *pd)
-{
- USED(fd);
- USED(pd);
- return 0;
-}
-
-int32
-runtime·netpollclose(uintptr fd)
-{
- USED(fd);
- return 0;
-}
-
-G*
-runtime·netpoll(bool block)
-{
- USED(block);
- return nil;
-}
diff --git a/src/pkg/runtime/netpoll_nacl.go b/src/pkg/runtime/netpoll_nacl.go
new file mode 100644
index 0000000..5cbc300
--- /dev/null
+++ b/src/pkg/runtime/netpoll_nacl.go
@@ -0,0 +1,26 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Fake network poller for NaCl.
+// Should never be used, because NaCl network connections do not honor "SetNonblock".
+
+package runtime
+
+func netpollinit() {
+}
+
+func netpollopen(fd uintptr, pd *pollDesc) int32 {
+ return 0
+}
+
+func netpollclose(fd uintptr) int32 {
+ return 0
+}
+
+func netpollarm(pd *pollDesc, mode int) {
+}
+
+func netpoll(block bool) *g {
+ return nil
+}
diff --git a/src/pkg/runtime/os_darwin.go b/src/pkg/runtime/os_darwin.go
index 4eefec1..e0f63dd 100644
--- a/src/pkg/runtime/os_darwin.go
+++ b/src/pkg/runtime/os_darwin.go
@@ -18,9 +18,6 @@
func sigaltstack(new, old unsafe.Pointer)
func sigtramp()
func setitimer(mode int32, new, old unsafe.Pointer)
-func kqueue() int32
-func kevent(fd int32, ev1 unsafe.Pointer, nev1 int32, ev2 unsafe.Pointer, nev2 int32, ts unsafe.Pointer) int32
-func closeonexec(fd int32)
func mach_semaphore_wait(sema uint32) int32
func mach_semaphore_timedwait(sema, sec, nsec uint32) int32
func mach_semaphore_signal(sema uint32) int32
diff --git a/src/pkg/runtime/os_dragonfly.go b/src/pkg/runtime/os_dragonfly.go
index 3e7fc8d..cdaa069 100644
--- a/src/pkg/runtime/os_dragonfly.go
+++ b/src/pkg/runtime/os_dragonfly.go
@@ -14,9 +14,6 @@
func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32
func getrlimit(kind int32, limit unsafe.Pointer) int32
func raise(sig int32)
-func kqueue() int32
-func kevent(fd int32, ev1 unsafe.Pointer, nev1 int32, ev2 unsafe.Pointer, nev2 int32, ts unsafe.Pointer) int32
-func closeonexec(fd int32)
func sys_umtx_sleep(addr unsafe.Pointer, val, timeout int32) int32
func sys_umtx_wakeup(addr unsafe.Pointer, val int32) int32
diff --git a/src/pkg/runtime/os_freebsd.go b/src/pkg/runtime/os_freebsd.go
index 14d1b7c..96964f1 100644
--- a/src/pkg/runtime/os_freebsd.go
+++ b/src/pkg/runtime/os_freebsd.go
@@ -14,9 +14,6 @@
func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32
func getrlimit(kind int32, limit unsafe.Pointer) int32
func raise(sig int32)
-func kqueue() int32
-func kevent(fd int32, ev1 unsafe.Pointer, nev1 int32, ev2 unsafe.Pointer, nev2 int32, ts unsafe.Pointer) int32
-func closeonexec(fd int32)
func sys_umtx_op(addr unsafe.Pointer, mode int32, val uint32, ptr2, ts unsafe.Pointer) int32
const stackSystem = 0
diff --git a/src/pkg/runtime/os_linux.go b/src/pkg/runtime/os_linux.go
index ba52a31..a6799cd 100644
--- a/src/pkg/runtime/os_linux.go
+++ b/src/pkg/runtime/os_linux.go
@@ -14,11 +14,6 @@
func rtsigprocmask(sig int32, new, old unsafe.Pointer, size int32)
func getrlimit(kind int32, limit unsafe.Pointer) int32
func raise(sig int32)
-func epollcreate(size int32) int32
-func epollcreate1(flags int32) int32
-func epollctl(epfd, op, fd int32, ev unsafe.Pointer) int32
-func epollwait(epfd int32, ev unsafe.Pointer, nev, timeout int32) int32
-func closeonexec(fd int32)
func sched_getaffinity(pid, len uintptr, buf *uintptr) int32
const stackSystem = 0
diff --git a/src/pkg/runtime/os_netbsd.go b/src/pkg/runtime/os_netbsd.go
index 1560768..8792f49 100644
--- a/src/pkg/runtime/os_netbsd.go
+++ b/src/pkg/runtime/os_netbsd.go
@@ -13,9 +13,6 @@
func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32
func lwp_tramp()
func raise(sig int32)
-func kqueue() int32
-func kevent(fd int32, ev1 unsafe.Pointer, nev1 int32, ev2 unsafe.Pointer, nev2 int32, ts unsafe.Pointer) int32
-func closeonexec(fd int32)
func getcontext(ctxt unsafe.Pointer)
func lwp_create(ctxt unsafe.Pointer, flags uintptr, lwpid unsafe.Pointer) int32
func lwp_park(abstime unsafe.Pointer, unpark int32, hint, unparkhint unsafe.Pointer) int32
diff --git a/src/pkg/runtime/os_openbsd.go b/src/pkg/runtime/os_openbsd.go
index 681bdde..19e2b45 100644
--- a/src/pkg/runtime/os_openbsd.go
+++ b/src/pkg/runtime/os_openbsd.go
@@ -12,9 +12,6 @@
func sigprocmask(mode int32, new uint32) uint32
func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32
func raise(sig int32)
-func kqueue() int32
-func kevent(fd int32, ev1 unsafe.Pointer, nev1 int32, ev2 unsafe.Pointer, nev2 int32, ts unsafe.Pointer) int32
-func closeonexec(fd int32)
func tfork(param unsafe.Pointer, psize uintptr, mm, gg, fn unsafe.Pointer) int32
func thrsleep(ident unsafe.Pointer, clock_id int32, tsp, lock, abort unsafe.Pointer) int32
func thrwakeup(ident unsafe.Pointer, n int32) int32
diff --git a/src/pkg/runtime/os_solaris.go b/src/pkg/runtime/os_solaris.go
index 556c150..a5e6966 100644
--- a/src/pkg/runtime/os_solaris.go
+++ b/src/pkg/runtime/os_solaris.go
@@ -19,6 +19,10 @@
func nanotime1() int64
func usleep1(usec uint32)
func osyield1()
+func netpollinit()
+func netpollopen(fd uintptr, pd *pollDesc) int32
+func netpollclose(fd uintptr) int32
+func netpollarm(pd *pollDesc, mode int)
type libcFunc byte
diff --git a/src/pkg/runtime/os_windows.go b/src/pkg/runtime/os_windows.go
index 15957b3..6a3bfca 100644
--- a/src/pkg/runtime/os_windows.go
+++ b/src/pkg/runtime/os_windows.go
@@ -21,6 +21,10 @@
func getlasterror() uint32
func setlasterror(err uint32)
func usleep1(usec uint32)
+func netpollinit()
+func netpollopen(fd uintptr, pd *pollDesc) int32
+func netpollclose(fd uintptr) int32
+func netpollarm(pd *pollDesc, mode int)
const stackSystem = 512 * ptrSize
diff --git a/src/pkg/runtime/runtime.h b/src/pkg/runtime/runtime.h
index 4cecc39..7007957 100644
--- a/src/pkg/runtime/runtime.h
+++ b/src/pkg/runtime/runtime.h
@@ -886,12 +886,8 @@
int64 runtime·tickspersecond(void);
void runtime·blockevent(int64, intgo);
G* runtime·netpoll(bool);
-void runtime·netpollinit(void);
-int32 runtime·netpollopen(uintptr, PollDesc*);
-int32 runtime·netpollclose(uintptr);
void runtime·netpollready(G**, PollDesc*, int32);
uintptr runtime·netpollfd(PollDesc*);
-void runtime·netpollarm(PollDesc*, int32);
void** runtime·netpolluser(PollDesc*);
bool runtime·netpollclosing(PollDesc*);
void runtime·netpolllock(PollDesc*);
diff --git a/src/pkg/runtime/stubs.go b/src/pkg/runtime/stubs.go
index 3719c75..73dc2ec 100644
--- a/src/pkg/runtime/stubs.go
+++ b/src/pkg/runtime/stubs.go
@@ -238,6 +238,9 @@
func xchgp(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer
//go:noescape
+func xchguintptr(ptr *uintptr, new uintptr) uintptr
+
+//go:noescape
func atomicstore(ptr *uint32, val uint32)
//go:noescape
diff --git a/src/pkg/runtime/sys_linux_386.s b/src/pkg/runtime/sys_linux_386.s
index ace5a18..426e05f 100644
--- a/src/pkg/runtime/sys_linux_386.s
+++ b/src/pkg/runtime/sys_linux_386.s
@@ -457,7 +457,7 @@
MOVL AX, ret+4(FP)
RET
-// int32 runtime·epollctl(int32 epfd, int32 op, int32 fd, EpollEvent *ev);
+// func epollctl(epfd, op, fd int32, ev *epollEvent) int
TEXT runtime·epollctl(SB),NOSPLIT,$0
MOVL $255, AX
MOVL epfd+0(FP), BX
diff --git a/src/pkg/runtime/sys_linux_amd64.s b/src/pkg/runtime/sys_linux_amd64.s
index f263ef3..3ce6bfc 100644
--- a/src/pkg/runtime/sys_linux_amd64.s
+++ b/src/pkg/runtime/sys_linux_amd64.s
@@ -378,7 +378,7 @@
MOVL AX, ret+8(FP)
RET
-// int32 runtime·epollctl(int32 epfd, int32 op, int32 fd, EpollEvent *ev);
+// func epollctl(epfd, op, fd int32, ev *epollEvent) int
TEXT runtime·epollctl(SB),NOSPLIT,$0
MOVL epfd+0(FP), DI
MOVL op+4(FP), SI
diff --git a/src/pkg/runtime/sys_linux_arm.s b/src/pkg/runtime/sys_linux_arm.s
index 3221cdf..a2abe8f 100644
--- a/src/pkg/runtime/sys_linux_arm.s
+++ b/src/pkg/runtime/sys_linux_arm.s
@@ -424,12 +424,12 @@
MOVW R0, ret+4(FP)
RET
-// int32 runtime·epollctl(int32 epfd, int32 op, int32 fd, EpollEvent *ev)
+// func epollctl(epfd, op, fd int32, ev *epollEvent) int
TEXT runtime·epollctl(SB),NOSPLIT,$0
- MOVW 0(FP), R0
- MOVW 4(FP), R1
- MOVW 8(FP), R2
- MOVW 12(FP), R3
+ MOVW epfd+0(FP), R0
+ MOVW op+4(FP), R1
+ MOVW fd+8(FP), R2
+ MOVW ev+12(FP), R3
MOVW $SYS_epoll_ctl, R7
SWI $0
MOVW R0, ret+16(FP)
diff --git a/src/pkg/runtime/thunk.s b/src/pkg/runtime/thunk.s
index 0b5963e..0c057a3 100644
--- a/src/pkg/runtime/thunk.s
+++ b/src/pkg/runtime/thunk.s
@@ -11,6 +11,9 @@
#define JMP B
#endif
+TEXT net·runtimeNano(SB),NOSPLIT,$0-0
+ JMP runtime·nanotime(SB)
+
TEXT time·runtimeNano(SB),NOSPLIT,$0-0
JMP runtime·nanotime(SB)
@@ -74,6 +77,30 @@
TEXT runtimeādebug·freeOSMemory(SB), NOSPLIT, $0-0
JMP runtime·freeOSMemory(SB)
+TEXT net·runtime_pollServerInit(SB),NOSPLIT,$0-0
+ JMP runtime·netpollServerInit(SB)
+
+TEXT net·runtime_pollOpen(SB),NOSPLIT,$0-0
+ JMP runtime·netpollOpen(SB)
+
+TEXT net·runtime_pollClose(SB),NOSPLIT,$0-0
+ JMP runtime·netpollClose(SB)
+
+TEXT net·runtime_pollReset(SB),NOSPLIT,$0-0
+ JMP runtime·netpollReset(SB)
+
+TEXT net·runtime_pollWait(SB),NOSPLIT,$0-0
+ JMP runtime·netpollWait(SB)
+
+TEXT net·runtime_pollWaitCanceled(SB),NOSPLIT,$0-0
+ JMP runtime·netpollWaitCanceled(SB)
+
+TEXT net·runtime_pollSetDeadline(SB),NOSPLIT,$0-0
+ JMP runtime·netpollSetDeadline(SB)
+
+TEXT net·runtime_pollUnblock(SB),NOSPLIT,$0-0
+ JMP runtime·netpollUnblock(SB)
+
TEXT syscall·setenv_c(SB), NOSPLIT, $0-0
JMP runtime·syscall_setenv_c(SB)
diff --git a/src/pkg/runtime/time.go b/src/pkg/runtime/time.go
index 102539b..8cf9eec 100644
--- a/src/pkg/runtime/time.go
+++ b/src/pkg/runtime/time.go
@@ -9,7 +9,7 @@
import "unsafe"
// Package time knows the layout of this structure.
-// If this struct changes, adjust ../time/sleep.go:/runtimeTimer and netpoll.goc:/timer.
+// If this struct changes, adjust ../time/sleep.go:/runtimeTimer.
// For GOOS=nacl, package syscall knows the layout of this structure.
// If this struct changes, adjust ../syscall/net_nacl.go:/runtimeTimer.
type timer struct {
@@ -20,8 +20,9 @@
// a well-behaved function and not block.
when int64
period int64
- f func(interface{})
+ f func(interface{}, uintptr)
arg interface{}
+ seq uintptr
}
var timers struct {
@@ -74,7 +75,7 @@
// Go runtime.
// Ready the goroutine arg.
-func goroutineReady(arg interface{}) {
+func goroutineReady(arg interface{}, seq uintptr) {
goready(arg.(*g))
}
@@ -185,11 +186,12 @@
}
f := t.f
arg := t.arg
+ seq := t.seq
unlock(&timers.lock)
if raceenabled {
raceacquire(unsafe.Pointer(t))
}
- f(arg)
+ f(arg, seq)
lock(&timers.lock)
}
if delta < 0 {
