| // Copyright 2013 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| package runtime |
| |
| import ( |
| "internal/goarch" |
| "internal/runtime/atomic" |
| "unsafe" |
| ) |
| |
| const _DWORD_MAX = 0xffffffff |
| |
| const _INVALID_HANDLE_VALUE = ^uintptr(0) |
| |
| // Sources are used to identify the event that created an overlapped entry. |
| // The source values are arbitrary. There is no risk of collision with user |
| // defined values because the only way to set the key of an overlapped entry |
| // is using the iocphandle, which is not accessible to user code. |
| const ( |
| netpollSourceReady = iota + 1 |
| netpollSourceBreak |
| netpollSourceTimer |
| ) |
| |
| const ( |
| // sourceBits is the number of bits needed to represent a source. |
| // 4 bits can hold 16 different sources, which is more than enough. |
| // It is set to a low value so the overlapped entry key can |
| // contain as much bits as possible for the pollDesc pointer. |
| sourceBits = 4 // 4 bits can hold 16 different sources, which is more than enough. |
| sourceMasks = 1<<sourceBits - 1 |
| ) |
| |
| // packNetpollKey creates a key from a source and a tag. |
| // Bits that don't fit in the result are discarded. |
| func packNetpollKey(source uint8, pd *pollDesc) uintptr { |
| // TODO: Consider combining the source with pd.fdseq to detect stale pollDescs. |
| if source > (1<<sourceBits)-1 { |
| // Also fail on 64-bit systems, even though it can hold more bits. |
| throw("runtime: source value is too large") |
| } |
| if goarch.PtrSize == 4 { |
| return uintptr(unsafe.Pointer(pd))<<sourceBits | uintptr(source) |
| } |
| return uintptr(taggedPointerPack(unsafe.Pointer(pd), uintptr(source))) |
| } |
| |
| // unpackNetpollSource returns the source packed key. |
| func unpackNetpollSource(key uintptr) uint8 { |
| if goarch.PtrSize == 4 { |
| return uint8(key & sourceMasks) |
| } |
| return uint8(taggedPointer(key).tag()) |
| } |
| |
| // pollOperation must be the same as beginning of internal/poll.operation. |
| // Keep these in sync. |
| type pollOperation struct { |
| // used by windows |
| _ overlapped |
| // used by netpoll |
| pd *pollDesc |
| mode int32 |
| } |
| |
| // pollOperationFromOverlappedEntry returns the pollOperation contained in |
| // e. It can return nil if the entry is not from internal/poll. |
| // See go.dev/issue/58870 |
| func pollOperationFromOverlappedEntry(e *overlappedEntry) *pollOperation { |
| if e.ov == nil { |
| return nil |
| } |
| op := (*pollOperation)(unsafe.Pointer(e.ov)) |
| // Check that the key matches the pollDesc pointer. |
| var keyMatch bool |
| if goarch.PtrSize == 4 { |
| keyMatch = e.key&^sourceMasks == uintptr(unsafe.Pointer(op.pd))<<sourceBits |
| } else { |
| keyMatch = (*pollDesc)(taggedPointer(e.key).pointer()) == op.pd |
| } |
| if !keyMatch { |
| return nil |
| } |
| return op |
| } |
| |
| // overlappedEntry contains the information returned by a call to GetQueuedCompletionStatusEx. |
| // https://learn.microsoft.com/en-us/windows/win32/api/minwinbase/ns-minwinbase-overlapped_entry |
| type overlappedEntry struct { |
| key uintptr |
| ov *overlapped |
| internal uintptr |
| qty uint32 |
| } |
| |
| var ( |
| iocphandle uintptr = _INVALID_HANDLE_VALUE // completion port io handle |
| |
| netpollWakeSig atomic.Uint32 // used to avoid duplicate calls of netpollBreak |
| ) |
| |
| func netpollinit() { |
| iocphandle = stdcall4(_CreateIoCompletionPort, _INVALID_HANDLE_VALUE, 0, 0, _DWORD_MAX) |
| if iocphandle == 0 { |
| println("runtime: CreateIoCompletionPort failed (errno=", getlasterror(), ")") |
| throw("runtime: netpollinit failed") |
| } |
| } |
| |
| func netpollIsPollDescriptor(fd uintptr) bool { |
| return fd == iocphandle |
| } |
| |
| func netpollopen(fd uintptr, pd *pollDesc) int32 { |
| key := packNetpollKey(netpollSourceReady, pd) |
| if stdcall4(_CreateIoCompletionPort, fd, iocphandle, key, 0) == 0 { |
| return int32(getlasterror()) |
| } |
| return 0 |
| } |
| |
| func netpollclose(fd uintptr) int32 { |
| // nothing to do |
| return 0 |
| } |
| |
| func netpollarm(pd *pollDesc, mode int) { |
| throw("runtime: unused") |
| } |
| |
| func netpollBreak() { |
| // Failing to cas indicates there is an in-flight wakeup, so we're done here. |
| if !netpollWakeSig.CompareAndSwap(0, 1) { |
| return |
| } |
| |
| key := packNetpollKey(netpollSourceBreak, nil) |
| if stdcall4(_PostQueuedCompletionStatus, iocphandle, 0, key, 0) == 0 { |
| println("runtime: netpoll: PostQueuedCompletionStatus failed (errno=", getlasterror(), ")") |
| throw("runtime: netpoll: PostQueuedCompletionStatus failed") |
| } |
| } |
| |
| // netpoll checks for ready network connections. |
| // Returns a list of goroutines that become runnable, |
| // and a delta to add to netpollWaiters. |
| // This must never return an empty list with a non-zero delta. |
| // |
| // delay < 0: blocks indefinitely |
| // delay == 0: does not block, just polls |
| // delay > 0: block for up to that many nanoseconds |
| func netpoll(delay int64) (gList, int32) { |
| if iocphandle == _INVALID_HANDLE_VALUE { |
| return gList{}, 0 |
| } |
| |
| var entries [64]overlappedEntry |
| var wait uint32 |
| var toRun gList |
| mp := getg().m |
| |
| if delay >= 1e15 { |
| // An arbitrary cap on how long to wait for a timer. |
| // 1e15 ns == ~11.5 days. |
| delay = 1e15 |
| } |
| |
| if delay > 0 && mp.waitIocpHandle != 0 { |
| // GetQueuedCompletionStatusEx doesn't use a high resolution timer internally, |
| // so we use a separate higher resolution timer associated with a wait completion |
| // packet to wake up the poller. Note that the completion packet can be delivered |
| // to another thread, and the Go scheduler expects netpoll to only block up to delay, |
| // so we still need to use a timeout with GetQueuedCompletionStatusEx. |
| // TODO: Improve the Go scheduler to support non-blocking timers. |
| signaled := netpollQueueTimer(delay) |
| if signaled { |
| // There is a small window between the SetWaitableTimer and the NtAssociateWaitCompletionPacket |
| // where the timer can expire. We can return immediately in this case. |
| return gList{}, 0 |
| } |
| } |
| if delay < 0 { |
| wait = _INFINITE |
| } else if delay == 0 { |
| wait = 0 |
| } else if delay < 1e6 { |
| wait = 1 |
| } else { |
| wait = uint32(delay / 1e6) |
| } |
| n := len(entries) / int(gomaxprocs) |
| if n < 8 { |
| n = 8 |
| } |
| if delay != 0 { |
| mp.blocked = true |
| } |
| if stdcall6(_GetQueuedCompletionStatusEx, iocphandle, uintptr(unsafe.Pointer(&entries[0])), uintptr(n), uintptr(unsafe.Pointer(&n)), uintptr(wait), 0) == 0 { |
| mp.blocked = false |
| errno := getlasterror() |
| if errno == _WAIT_TIMEOUT { |
| return gList{}, 0 |
| } |
| println("runtime: GetQueuedCompletionStatusEx failed (errno=", errno, ")") |
| throw("runtime: netpoll failed") |
| } |
| mp.blocked = false |
| delta := int32(0) |
| for i := 0; i < n; i++ { |
| e := &entries[i] |
| switch unpackNetpollSource(e.key) { |
| case netpollSourceReady: |
| op := pollOperationFromOverlappedEntry(e) |
| if op == nil { |
| // Entry from outside the Go runtime and internal/poll, ignore. |
| continue |
| } |
| // Entry from internal/poll. |
| mode := op.mode |
| if mode != 'r' && mode != 'w' { |
| println("runtime: GetQueuedCompletionStatusEx returned net_op with invalid mode=", mode) |
| throw("runtime: netpoll failed") |
| } |
| delta += netpollready(&toRun, op.pd, mode) |
| case netpollSourceBreak: |
| netpollWakeSig.Store(0) |
| if delay == 0 { |
| // Forward the notification to the blocked poller. |
| netpollBreak() |
| } |
| case netpollSourceTimer: |
| // TODO: We could avoid calling NtCancelWaitCompletionPacket for expired wait completion packets. |
| default: |
| println("runtime: GetQueuedCompletionStatusEx returned net_op with invalid key=", e.key) |
| throw("runtime: netpoll failed") |
| } |
| } |
| return toRun, delta |
| } |
| |
| // netpollQueueTimer queues a timer to wake up the poller after the given delay. |
| // It returns true if the timer expired during this call. |
| func netpollQueueTimer(delay int64) (signaled bool) { |
| const ( |
| STATUS_SUCCESS = 0x00000000 |
| STATUS_PENDING = 0x00000103 |
| STATUS_CANCELLED = 0xC0000120 |
| ) |
| mp := getg().m |
| // A wait completion packet can only be associated with one timer at a time, |
| // so we need to cancel the previous one if it exists. This wouldn't be necessary |
| // if the poller would only be woken up by the timer, in which case the association |
| // would be automatically canceled, but it can also be woken up by other events, |
| // such as a netpollBreak, so we can get to this point with a timer that hasn't |
| // expired yet. In this case, the completion packet can still be picked up by |
| // another thread, so defer the cancellation until it is really necessary. |
| errno := stdcall2(_NtCancelWaitCompletionPacket, mp.waitIocpHandle, 1) |
| switch errno { |
| case STATUS_CANCELLED: |
| // STATUS_CANCELLED is returned when the associated timer has already expired, |
| // in which automatically cancels the wait completion packet. |
| fallthrough |
| case STATUS_SUCCESS: |
| dt := -delay / 100 // relative sleep (negative), 100ns units |
| if stdcall6(_SetWaitableTimer, mp.waitIocpTimer, uintptr(unsafe.Pointer(&dt)), 0, 0, 0, 0) == 0 { |
| println("runtime: SetWaitableTimer failed; errno=", getlasterror()) |
| throw("runtime: netpoll failed") |
| } |
| key := packNetpollKey(netpollSourceTimer, nil) |
| if errno := stdcall8(_NtAssociateWaitCompletionPacket, mp.waitIocpHandle, iocphandle, mp.waitIocpTimer, key, 0, 0, 0, uintptr(unsafe.Pointer(&signaled))); errno != 0 { |
| println("runtime: NtAssociateWaitCompletionPacket failed; errno=", errno) |
| throw("runtime: netpoll failed") |
| } |
| case STATUS_PENDING: |
| // STATUS_PENDING is returned if the wait operation can't be canceled yet. |
| // This can happen if this thread was woken up by another event, such as a netpollBreak, |
| // and the timer expired just while calling NtCancelWaitCompletionPacket, in which case |
| // this call fails to cancel the association to avoid a race condition. |
| // This is a rare case, so we can just avoid using the high resolution timer this time. |
| default: |
| println("runtime: NtCancelWaitCompletionPacket failed; errno=", errno) |
| throw("runtime: netpoll failed") |
| } |
| return signaled |
| } |