Skip to content

Commit 41b4298

Browse files
mknyszekgopherbot
mknyszek
authored and
gopherbot
committed
runtime: duplicate scanobject in greentea and non-greentea files
This change exists to help differentiate profile samples spent on Green Tea and non-Green-Tea GC time in mixed contexts. Change-Id: I8dea340d2d11ba4c410ae939fb5f37020d0b55d1 Reviewed-on: https://go-review.googlesource.com/c/go/+/689477 Reviewed-by: Michael Pratt <[email protected]> Auto-Submit: Michael Knyszek <[email protected]> LUCI-TryBot-Result: Go LUCI <[email protected]>
1 parent aeb256e commit 41b4298

File tree

3 files changed

+211
-102
lines changed

3 files changed

+211
-102
lines changed

‎src/runtime/mgcmark.go

Lines changed: 0 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -1435,107 +1435,6 @@ func scanblock(b0, n0 uintptr, ptrmask *uint8, gcw *gcWork, stk *stackScanState)
14351435
}
14361436
}
14371437

1438-
// scanobject scans the object starting at b, adding pointers to gcw.
1439-
// b must point to the beginning of a heap object or an oblet.
1440-
// scanobject consults the GC bitmap for the pointer mask and the
1441-
// spans for the size of the object.
1442-
//
1443-
//go:nowritebarrier
1444-
func scanobject(b uintptr, gcw *gcWork) {
1445-
// Prefetch object before we scan it.
1446-
//
1447-
// This will overlap fetching the beginning of the object with initial
1448-
// setup before we start scanning the object.
1449-
sys.Prefetch(b)
1450-
1451-
// Find the bits for b and the size of the object at b.
1452-
//
1453-
// b is either the beginning of an object, in which case this
1454-
// is the size of the object to scan, or it points to an
1455-
// oblet, in which case we compute the size to scan below.
1456-
s := spanOfUnchecked(b)
1457-
n := s.elemsize
1458-
if n == 0 {
1459-
throw("scanobject n == 0")
1460-
}
1461-
if s.spanclass.noscan() {
1462-
// Correctness-wise this is ok, but it's inefficient
1463-
// if noscan objects reach here.
1464-
throw("scanobject of a noscan object")
1465-
}
1466-
1467-
var tp typePointers
1468-
if n > maxObletBytes {
1469-
// Large object. Break into oblets for better
1470-
// parallelism and lower latency.
1471-
if b == s.base() {
1472-
// Enqueue the other oblets to scan later.
1473-
// Some oblets may be in b's scalar tail, but
1474-
// these will be marked as "no more pointers",
1475-
// so we'll drop out immediately when we go to
1476-
// scan those.
1477-
for oblet := b + maxObletBytes; oblet < s.base()+s.elemsize; oblet += maxObletBytes {
1478-
if !gcw.putObjFast(oblet) {
1479-
gcw.putObj(oblet)
1480-
}
1481-
}
1482-
}
1483-
1484-
// Compute the size of the oblet. Since this object
1485-
// must be a large object, s.base() is the beginning
1486-
// of the object.
1487-
n = s.base() + s.elemsize - b
1488-
n = min(n, maxObletBytes)
1489-
tp = s.typePointersOfUnchecked(s.base())
1490-
tp = tp.fastForward(b-tp.addr, b+n)
1491-
} else {
1492-
tp = s.typePointersOfUnchecked(b)
1493-
}
1494-
1495-
var scanSize uintptr
1496-
for {
1497-
var addr uintptr
1498-
if tp, addr = tp.nextFast(); addr == 0 {
1499-
if tp, addr = tp.next(b + n); addr == 0 {
1500-
break
1501-
}
1502-
}
1503-
1504-
// Keep track of farthest pointer we found, so we can
1505-
// update heapScanWork. TODO: is there a better metric,
1506-
// now that we can skip scalar portions pretty efficiently?
1507-
scanSize = addr - b + goarch.PtrSize
1508-
1509-
// Work here is duplicated in scanblock and above.
1510-
// If you make changes here, make changes there too.
1511-
obj := *(*uintptr)(unsafe.Pointer(addr))
1512-
1513-
// At this point we have extracted the next potential pointer.
1514-
// Quickly filter out nil and pointers back to the current object.
1515-
if obj != 0 && obj-b >= n {
1516-
// Test if obj points into the Go heap and, if so,
1517-
// mark the object.
1518-
//
1519-
// Note that it's possible for findObject to
1520-
// fail if obj points to a just-allocated heap
1521-
// object because of a race with growing the
1522-
// heap. In this case, we know the object was
1523-
// just allocated and hence will be marked by
1524-
// allocation itself.
1525-
if !tryDeferToSpanScan(obj, gcw) {
1526-
if obj, span, objIndex := findObject(obj, b, addr-b); obj != 0 {
1527-
greyobject(obj, b, addr-b, span, gcw, objIndex)
1528-
}
1529-
}
1530-
}
1531-
}
1532-
gcw.bytesMarked += uint64(n)
1533-
gcw.heapScanWork += int64(scanSize)
1534-
if debug.gctrace > 1 {
1535-
gcw.stats[s.spanclass.sizeclass()].sparseObjsScanned++
1536-
}
1537-
}
1538-
15391438
// scanConservative scans block [b, b+n) conservatively, treating any
15401439
// pointer-like value in the block as a pointer.
15411440
//

‎src/runtime/mgcmark_greenteagc.go

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -851,3 +851,107 @@ func (w *gcWork) flushScanStats(dst *[gc.NumSizeClasses]sizeClassScanStats) {
851851
}
852852
clear(w.stats[:])
853853
}
854+
855+
// scanobject scans the object starting at b, adding pointers to gcw.
856+
// b must point to the beginning of a heap object or an oblet.
857+
// scanobject consults the GC bitmap for the pointer mask and the
858+
// spans for the size of the object.
859+
//
860+
// Used only for !gcUsesSpanInlineMarkBits spans, but supports all
861+
// object sizes and is safe to be called on all heap objects.
862+
//
863+
//go:nowritebarrier
864+
func scanobject(b uintptr, gcw *gcWork) {
865+
// Prefetch object before we scan it.
866+
//
867+
// This will overlap fetching the beginning of the object with initial
868+
// setup before we start scanning the object.
869+
sys.Prefetch(b)
870+
871+
// Find the bits for b and the size of the object at b.
872+
//
873+
// b is either the beginning of an object, in which case this
874+
// is the size of the object to scan, or it points to an
875+
// oblet, in which case we compute the size to scan below.
876+
s := spanOfUnchecked(b)
877+
n := s.elemsize
878+
if n == 0 {
879+
throw("scanobject n == 0")
880+
}
881+
if s.spanclass.noscan() {
882+
// Correctness-wise this is ok, but it's inefficient
883+
// if noscan objects reach here.
884+
throw("scanobject of a noscan object")
885+
}
886+
887+
var tp typePointers
888+
if n > maxObletBytes {
889+
// Large object. Break into oblets for better
890+
// parallelism and lower latency.
891+
if b == s.base() {
892+
// Enqueue the other oblets to scan later.
893+
// Some oblets may be in b's scalar tail, but
894+
// these will be marked as "no more pointers",
895+
// so we'll drop out immediately when we go to
896+
// scan those.
897+
for oblet := b + maxObletBytes; oblet < s.base()+s.elemsize; oblet += maxObletBytes {
898+
if !gcw.putObjFast(oblet) {
899+
gcw.putObj(oblet)
900+
}
901+
}
902+
}
903+
904+
// Compute the size of the oblet. Since this object
905+
// must be a large object, s.base() is the beginning
906+
// of the object.
907+
n = s.base() + s.elemsize - b
908+
n = min(n, maxObletBytes)
909+
tp = s.typePointersOfUnchecked(s.base())
910+
tp = tp.fastForward(b-tp.addr, b+n)
911+
} else {
912+
tp = s.typePointersOfUnchecked(b)
913+
}
914+
915+
var scanSize uintptr
916+
for {
917+
var addr uintptr
918+
if tp, addr = tp.nextFast(); addr == 0 {
919+
if tp, addr = tp.next(b + n); addr == 0 {
920+
break
921+
}
922+
}
923+
924+
// Keep track of farthest pointer we found, so we can
925+
// update heapScanWork. TODO: is there a better metric,
926+
// now that we can skip scalar portions pretty efficiently?
927+
scanSize = addr - b + goarch.PtrSize
928+
929+
// Work here is duplicated in scanblock and above.
930+
// If you make changes here, make changes there too.
931+
obj := *(*uintptr)(unsafe.Pointer(addr))
932+
933+
// At this point we have extracted the next potential pointer.
934+
// Quickly filter out nil and pointers back to the current object.
935+
if obj != 0 && obj-b >= n {
936+
// Test if obj points into the Go heap and, if so,
937+
// mark the object.
938+
//
939+
// Note that it's possible for findObject to
940+
// fail if obj points to a just-allocated heap
941+
// object because of a race with growing the
942+
// heap. In this case, we know the object was
943+
// just allocated and hence will be marked by
944+
// allocation itself.
945+
if !tryDeferToSpanScan(obj, gcw) {
946+
if obj, span, objIndex := findObject(obj, b, addr-b); obj != 0 {
947+
greyobject(obj, b, addr-b, span, gcw, objIndex)
948+
}
949+
}
950+
}
951+
}
952+
gcw.bytesMarked += uint64(n)
953+
gcw.heapScanWork += int64(scanSize)
954+
if debug.gctrace > 1 {
955+
gcw.stats[s.spanclass.sizeclass()].sparseObjsScanned++
956+
}
957+
}

‎src/runtime/mgcmark_nogreenteagc.go

Lines changed: 107 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,12 @@
66

77
package runtime
88

9-
import "internal/runtime/gc"
9+
import (
10+
"internal/goarch"
11+
"internal/runtime/gc"
12+
"internal/runtime/sys"
13+
"unsafe"
14+
)
1015

1116
func (s *mspan) markBitsForIndex(objIndex uintptr) markBits {
1217
bytep, mask := s.gcmarkBits.bitp(objIndex)
@@ -110,3 +115,104 @@ func (w *gcWork) flushScanStats(dst *[gc.NumSizeClasses]sizeClassScanStats) {
110115
}
111116
clear(w.stats[:])
112117
}
118+
119+
// scanobject scans the object starting at b, adding pointers to gcw.
120+
// b must point to the beginning of a heap object or an oblet.
121+
// scanobject consults the GC bitmap for the pointer mask and the
122+
// spans for the size of the object.
123+
//
124+
//go:nowritebarrier
125+
func scanobject(b uintptr, gcw *gcWork) {
126+
// Prefetch object before we scan it.
127+
//
128+
// This will overlap fetching the beginning of the object with initial
129+
// setup before we start scanning the object.
130+
sys.Prefetch(b)
131+
132+
// Find the bits for b and the size of the object at b.
133+
//
134+
// b is either the beginning of an object, in which case this
135+
// is the size of the object to scan, or it points to an
136+
// oblet, in which case we compute the size to scan below.
137+
s := spanOfUnchecked(b)
138+
n := s.elemsize
139+
if n == 0 {
140+
throw("scanobject n == 0")
141+
}
142+
if s.spanclass.noscan() {
143+
// Correctness-wise this is ok, but it's inefficient
144+
// if noscan objects reach here.
145+
throw("scanobject of a noscan object")
146+
}
147+
148+
var tp typePointers
149+
if n > maxObletBytes {
150+
// Large object. Break into oblets for better
151+
// parallelism and lower latency.
152+
if b == s.base() {
153+
// Enqueue the other oblets to scan later.
154+
// Some oblets may be in b's scalar tail, but
155+
// these will be marked as "no more pointers",
156+
// so we'll drop out immediately when we go to
157+
// scan those.
158+
for oblet := b + maxObletBytes; oblet < s.base()+s.elemsize; oblet += maxObletBytes {
159+
if !gcw.putObjFast(oblet) {
160+
gcw.putObj(oblet)
161+
}
162+
}
163+
}
164+
165+
// Compute the size of the oblet. Since this object
166+
// must be a large object, s.base() is the beginning
167+
// of the object.
168+
n = s.base() + s.elemsize - b
169+
n = min(n, maxObletBytes)
170+
tp = s.typePointersOfUnchecked(s.base())
171+
tp = tp.fastForward(b-tp.addr, b+n)
172+
} else {
173+
tp = s.typePointersOfUnchecked(b)
174+
}
175+
176+
var scanSize uintptr
177+
for {
178+
var addr uintptr
179+
if tp, addr = tp.nextFast(); addr == 0 {
180+
if tp, addr = tp.next(b + n); addr == 0 {
181+
break
182+
}
183+
}
184+
185+
// Keep track of farthest pointer we found, so we can
186+
// update heapScanWork. TODO: is there a better metric,
187+
// now that we can skip scalar portions pretty efficiently?
188+
scanSize = addr - b + goarch.PtrSize
189+
190+
// Work here is duplicated in scanblock and above.
191+
// If you make changes here, make changes there too.
192+
obj := *(*uintptr)(unsafe.Pointer(addr))
193+
194+
// At this point we have extracted the next potential pointer.
195+
// Quickly filter out nil and pointers back to the current object.
196+
if obj != 0 && obj-b >= n {
197+
// Test if obj points into the Go heap and, if so,
198+
// mark the object.
199+
//
200+
// Note that it's possible for findObject to
201+
// fail if obj points to a just-allocated heap
202+
// object because of a race with growing the
203+
// heap. In this case, we know the object was
204+
// just allocated and hence will be marked by
205+
// allocation itself.
206+
if !tryDeferToSpanScan(obj, gcw) {
207+
if obj, span, objIndex := findObject(obj, b, addr-b); obj != 0 {
208+
greyobject(obj, b, addr-b, span, gcw, objIndex)
209+
}
210+
}
211+
}
212+
}
213+
gcw.bytesMarked += uint64(n)
214+
gcw.heapScanWork += int64(scanSize)
215+
if debug.gctrace > 1 {
216+
gcw.stats[s.spanclass.sizeclass()].sparseObjsScanned++
217+
}
218+
}

0 commit comments

Comments
 (0)