commit 016d5eea118445f5b3ca46f85b65c6afd180c946
author Michael Anthony Knyszek <mknyszek@google.com> Mon Oct 18 23:14:20 2021 +0000
committer Michael Knyszek <mknyszek@google.com> Wed Oct 20 20:39:36 2021 +0000
tree c708523cf135857f9513783066ebf818d5be82f7
parent a91e976fd2cc6f1c156877eae40262d7e47c745c

runtime: retype mheap.reclaimCredit as atomic.Uintptr

[git-generate]
cd src/runtime
mv export_test.go export.go
GOROOT=$(dirname $(dirname $PWD)) rf '
  add mheap.reclaimCredit \
// reclaimCredit is spare credit for extra pages swept. Since \
// the page reclaimer works in large chunks, it may reclaim \
// more than requested. Any spare pages released go to this \
// credit pool. \
reclaimCredit_ atomic.Uintptr
  ex {
    import "runtime/internal/atomic"

    var t mheap
    var v, w uintptr
    var d uintptr

    t.reclaimCredit -> t.reclaimCredit_.Load()
    t.reclaimCredit = v -> t.reclaimCredit_.Store(v)
    atomic.Loaduintptr(&t.reclaimCredit) -> t.reclaimCredit_.Load()
    atomic.LoadAcquintptr(&t.reclaimCredit) -> t.reclaimCredit_.LoadAcquire()
    atomic.Storeuintptr(&t.reclaimCredit, v) -> t.reclaimCredit_.Store(v)
    atomic.StoreReluintptr(&t.reclaimCredit, v) -> t.reclaimCredit_.StoreRelease(v)
    atomic.Casuintptr(&t.reclaimCredit, v, w) -> t.reclaimCredit_.CompareAndSwap(v, w)
    atomic.Xchguintptr(&t.reclaimCredit, v) -> t.reclaimCredit_.Swap(v)
    atomic.Xadduintptr(&t.reclaimCredit, d) -> t.reclaimCredit_.Add(d)
  }
  rm mheap.reclaimCredit
  mv mheap.reclaimCredit_ mheap.reclaimCredit
'
mv export.go export_test.go

Change-Id: I2c567781a28f5d8c2275ff18f2cf605b82f22d09
Reviewed-on: https://go-review.googlesource.com/c/go/+/356712
Trust: Michael Knyszek <mknyszek@google.com>
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>

src/runtime/mgc.go

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index 56ef1c4..6f8463c 100644
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -1460,7 +1460,7 @@
 	mheap_.pagesSwept.Store(0)
 	mheap_.sweepArenas = mheap_.allArenas
 	mheap_.reclaimIndex.Store(0)
-	mheap_.reclaimCredit = 0
+	mheap_.reclaimCredit.Store(0)
 	unlock(&mheap_.lock)
 
 	sweep.centralIndex.clear()

src/runtime/mgcsweep.go

diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go
index aedd6c3..9c7f9d3 100644
--- a/src/runtime/mgcsweep.go
+++ b/src/runtime/mgcsweep.go
@@ -291,7 +291,7 @@
 				// Whole span was freed. Count it toward the
 				// page reclaimer credit since these pages can
 				// now be used for span allocation.
-				atomic.Xadduintptr(&mheap_.reclaimCredit, npages)
+				mheap_.reclaimCredit.Add(npages)
 			} else {
 				// Span is still in-use, so this returned no
 				// pages to the heap and the span needs to

src/runtime/mheap.go

diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go
index fc86023..0e7694a 100644
--- a/src/runtime/mheap.go
+++ b/src/runtime/mheap.go
@@ -124,13 +124,12 @@
 	// If this is >= 1<<63, the page reclaimer is done scanning
 	// the page marks.
 	reclaimIndex atomic.Uint64
+
 	// reclaimCredit is spare credit for extra pages swept. Since
 	// the page reclaimer works in large chunks, it may reclaim
 	// more than requested. Any spare pages released go to this
 	// credit pool.
-	//
-	// This is accessed atomically.
-	reclaimCredit uintptr
+	reclaimCredit atomic.Uintptr
 
 	// arenas is the heap arena map. It points to the metadata for
 	// the heap for every arena frame of the entire usable virtual
@@ -754,13 +753,13 @@
 	locked := false
 	for npage > 0 {
 		// Pull from accumulated credit first.
-		if credit := atomic.Loaduintptr(&h.reclaimCredit); credit > 0 {
+		if credit := h.reclaimCredit.Load(); credit > 0 {
 			take := credit
 			if take > npage {
 				// Take only what we need.
 				take = npage
 			}
-			if atomic.Casuintptr(&h.reclaimCredit, credit, credit-take) {
+			if h.reclaimCredit.CompareAndSwap(credit, credit-take) {
 				npage -= take
 			}
 			continue
@@ -786,7 +785,7 @@
 			npage -= nfound
 		} else {
 			// Put spare pages toward global credit.
-			atomic.Xadduintptr(&h.reclaimCredit, nfound-npage)
+			h.reclaimCredit.Add(nfound - npage)
 			npage = 0
 		}
 	}