runtime: preempt fractional worker after reaching utilization goal
Currently fractional workers run until preempted by the scheduler,
which means they typically run for 20ms. During this time, all other
goroutines on that P are blocked, which can introduce significant
latency variance.
This modifies fractional workers to self-preempt shortly after
achieving the fractional utilization goal. In practice this means they
preempt much sooner, and the scale of their preemption is on the order
of how often the user goroutine block (so, if the application is
compute-bound, the fractional workers will also run for long times,
but if the application blocks frequently, the fractional workers will
also preempt quickly).
Fixes #21698.
Updates #18534.
Change-Id: I03a5ab195dae93154a46c32083c4bb52415d2017
Reviewed-on: https://go-review.googlesource.com/68573
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Rick Hudson <rlh@golang.org>
diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go
index 016c1f7..ed256ef 100644
--- a/src/runtime/mgcmark.go
+++ b/src/runtime/mgcmark.go
@@ -34,13 +34,13 @@
// span base.
maxObletBytes = 128 << 10
- // idleCheckThreshold specifies how many units of work to do
- // between run queue checks in an idle worker. Assuming a scan
+ // drainCheckThreshold specifies how many units of work to do
+ // between self-preemption checks in gcDrain. Assuming a scan
// rate of 1 MB/ms, this is ~100 µs. Lower values have higher
// overhead in the scan loop (the scheduler check may perform
// a syscall, so its overhead is nontrivial). Higher values
// make the system less responsive to incoming work.
- idleCheckThreshold = 100000
+ drainCheckThreshold = 100000
)
// gcMarkRootPrepare queues root scanning jobs (stacks, globals, and
@@ -861,6 +861,7 @@
gcDrainNoBlock
gcDrainFlushBgCredit
gcDrainIdle
+ gcDrainFractional
// gcDrainBlock means neither gcDrainUntilPreempt or
// gcDrainNoBlock. It is the default, but callers should use
@@ -877,6 +878,10 @@
// If flags&gcDrainIdle != 0, gcDrain returns when there is other work
// to do. This implies gcDrainNoBlock.
//
+// If flags&gcDrainFractional != 0, gcDrain self-preempts when
+// pollFractionalWorkerExit() returns true. This implies
+// gcDrainNoBlock.
+//
// If flags&gcDrainNoBlock != 0, gcDrain returns as soon as it is
// unable to get more work. Otherwise, it will block until all
// blocking calls are blocked in gcDrain.
@@ -893,14 +898,24 @@
gp := getg().m.curg
preemptible := flags&gcDrainUntilPreempt != 0
- blocking := flags&(gcDrainUntilPreempt|gcDrainIdle|gcDrainNoBlock) == 0
+ blocking := flags&(gcDrainUntilPreempt|gcDrainIdle|gcDrainFractional|gcDrainNoBlock) == 0
flushBgCredit := flags&gcDrainFlushBgCredit != 0
idle := flags&gcDrainIdle != 0
initScanWork := gcw.scanWork
- // idleCheck is the scan work at which to perform the next
- // idle check with the scheduler.
- idleCheck := initScanWork + idleCheckThreshold
+
+ // checkWork is the scan work before performing the next
+ // self-preempt check.
+ checkWork := int64(1<<63 - 1)
+ var check func() bool
+ if flags&(gcDrainIdle|gcDrainFractional) != 0 {
+ checkWork = initScanWork + drainCheckThreshold
+ if idle {
+ check = pollWork
+ } else if flags&gcDrainFractional != 0 {
+ check = pollFractionalWorkerExit
+ }
+ }
// Drain root marking jobs.
if work.markrootNext < work.markrootJobs {
@@ -910,7 +925,7 @@
break
}
markroot(gcw, job)
- if idle && pollWork() {
+ if check != nil && check() {
goto done
}
}
@@ -951,12 +966,12 @@
gcFlushBgCredit(gcw.scanWork - initScanWork)
initScanWork = 0
}
- idleCheck -= gcw.scanWork
+ checkWork -= gcw.scanWork
gcw.scanWork = 0
- if idle && idleCheck <= 0 {
- idleCheck += idleCheckThreshold
- if pollWork() {
+ if checkWork <= 0 {
+ checkWork += drainCheckThreshold
+ if check != nil && check() {
break
}
}
