runtime: convert slice operations to Go.
LGTM=bradfitz, dvyukov
R=golang-codereviews, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/120190044
diff --git a/src/pkg/runtime/alg.goc b/src/pkg/runtime/alg.goc
index 3db1456..41be9c0 100644
--- a/src/pkg/runtime/alg.goc
+++ b/src/pkg/runtime/alg.goc
@@ -286,7 +286,7 @@
}
void
-runtime·slicecopy(uintptr s, void *a, void *b)
+runtime·algslicecopy(uintptr s, void *a, void *b)
{
USED(s);
if(b == nil) {
@@ -445,7 +445,7 @@
[ASTRING] { runtime·strhash, runtime·strequal, runtime·strprint, runtime·strcopy },
[AINTER] { runtime·interhash, runtime·interequal, runtime·interprint, runtime·intercopy },
[ANILINTER] { runtime·nilinterhash, runtime·nilinterequal, runtime·nilinterprint, runtime·nilintercopy },
-[ASLICE] { runtime·nohash, runtime·noequal, runtime·memprint, runtime·slicecopy },
+[ASLICE] { runtime·nohash, runtime·noequal, runtime·memprint, runtime·algslicecopy },
[AFLOAT32] { runtime·f32hash, runtime·f32equal, runtime·memprint, runtime·memcopy },
[AFLOAT64] { runtime·f64hash, runtime·f64equal, runtime·memprint, runtime·memcopy },
[ACPLX64] { runtime·c64hash, runtime·c64equal, runtime·memprint, runtime·memcopy },
diff --git a/src/pkg/runtime/malloc.go b/src/pkg/runtime/malloc.go
index cac8f96..255778b 100644
--- a/src/pkg/runtime/malloc.go
+++ b/src/pkg/runtime/malloc.go
@@ -232,6 +232,12 @@
return gomallocgc(uintptr(typ.size)*n, typ, flags)
}
+// rawmem returns a chunk of pointerless memory. It is
+// not zeroed.
+func rawmem(size uintptr) unsafe.Pointer {
+ return gomallocgc(size, nil, flagNoScan|flagNoZero)
+}
+
// round size up to next size class
func goroundupsize(size uintptr) uintptr {
if size < maxSmallSize {
diff --git a/src/pkg/runtime/slice.c b/src/pkg/runtime/slice.c
new file mode 100644
index 0000000..5483a20
--- /dev/null
+++ b/src/pkg/runtime/slice.c
@@ -0,0 +1,27 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "runtime.h"
+#include "arch_GOARCH.h"
+
+void
+runtime·printslice_m(G *gp)
+{
+ void *array;
+ uintptr len, cap;
+
+ array = g->m->ptrarg[0];
+ g->m->ptrarg[0] = nil;
+ len = g->m->scalararg[0];
+ cap = g->m->scalararg[1];
+
+ runtime·prints("[");
+ runtime·printint(len);
+ runtime·prints("/");
+ runtime·printint(cap);
+ runtime·prints("]");
+ runtime·printpointer(array);
+
+ runtime·gogo(&gp->sched);
+}
diff --git a/src/pkg/runtime/slice.go b/src/pkg/runtime/slice.go
new file mode 100644
index 0000000..6ed7068
--- /dev/null
+++ b/src/pkg/runtime/slice.go
@@ -0,0 +1,154 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime
+
+import (
+ "unsafe"
+)
+
+type sliceStruct struct {
+ array unsafe.Pointer
+ len int
+ cap int
+}
+
+// TODO: take uintptrs instead of int64s?
+func makeslice(t *slicetype, len64 int64, cap64 int64) sliceStruct {
+ // NOTE: The len > MaxMem/elemsize check here is not strictly necessary,
+ // but it produces a 'len out of range' error instead of a 'cap out of range' error
+ // when someone does make([]T, bignumber). 'cap out of range' is true too,
+ // but since the cap is only being supplied implicitly, saying len is clearer.
+ // See issue 4085.
+ len := int(len64)
+ if len64 < 0 || int64(len) != len64 || t.elem.size > 0 && len > int(maxMem/uintptr(t.elem.size)) {
+ panic(errorString("makeslice: len out of range"))
+ }
+ cap := int(cap64)
+ if cap < len || int64(cap) != cap64 || t.elem.size > 0 && cap > int(maxMem/uintptr(t.elem.size)) {
+ panic(errorString("makeslice: cap out of range"))
+ }
+ p := newarray(t.elem, uintptr(cap))
+ return sliceStruct{p, len, cap}
+}
+
+// TODO: take uintptr instead of int64?
+func growslice(t *slicetype, old sliceStruct, n int64) sliceStruct {
+ if n < 1 {
+ panic(errorString("growslice: invalid n"))
+ }
+
+ cap64 := int64(old.cap) + n
+ cap := int(cap64)
+
+ if int64(cap) != cap64 || cap < old.cap || t.elem.size > 0 && cap > int(maxMem/uintptr(t.elem.size)) {
+ panic(errorString("growslice: cap out of range"))
+ }
+
+ if raceenabled {
+ callerpc := gogetcallerpc(unsafe.Pointer(&t))
+ fn := growslice
+ pc := **(**uintptr)(unsafe.Pointer(&fn))
+ racereadrangepc(old.array, old.len*int(t.elem.size), callerpc, pc)
+ }
+
+ et := t.elem
+ if et.size == 0 {
+ return sliceStruct{old.array, old.len, cap}
+ }
+
+ newcap := old.cap
+ if newcap+newcap < cap {
+ newcap = cap
+ } else {
+ for {
+ if old.len < 1024 {
+ newcap += newcap
+ } else {
+ newcap += newcap / 4
+ }
+ if newcap >= cap {
+ break
+ }
+ }
+ }
+
+ if newcap >= int(maxMem/uintptr(et.size)) {
+ panic(errorString("growslice: cap out of range"))
+ }
+ lenmem := uintptr(old.len) * uintptr(et.size)
+ capmem := goroundupsize(uintptr(newcap) * uintptr(et.size))
+ newcap = int(capmem / uintptr(et.size))
+ var p unsafe.Pointer
+ if et.kind&kindNoPointers != 0 {
+ p = rawmem(capmem)
+ memclr(add(p, lenmem), capmem-lenmem)
+ } else {
+ // Note: can't use rawmem (which avoids zeroing of memory), because then GC can scan unitialized memory
+ p = newarray(et, uintptr(newcap))
+ }
+ memmove(p, old.array, lenmem)
+
+ return sliceStruct{p, old.len, newcap}
+}
+
+func slicecopy(to sliceStruct, fm sliceStruct, width uintptr) int {
+ if fm.len == 0 || to.len == 0 || width == 0 {
+ return 0
+ }
+
+ n := fm.len
+ if to.len < n {
+ n = to.len
+ }
+
+ if raceenabled {
+ callerpc := gogetcallerpc(unsafe.Pointer(&to))
+ fn := slicecopy
+ pc := **(**uintptr)(unsafe.Pointer(&fn))
+ racewriterangepc(to.array, n*int(width), callerpc, pc)
+ racereadrangepc(fm.array, n*int(width), callerpc, pc)
+ }
+
+ size := uintptr(n) * width
+ if size == 1 { // common case worth about 2x to do here
+ // TODO: is this still worth it with new memmove impl?
+ *(*byte)(to.array) = *(*byte)(fm.array) // known to be a byte pointer
+ } else {
+ memmove(to.array, fm.array, size)
+ }
+ return int(n)
+}
+
+func slicestringcopy(to []byte, fm string) int {
+ if len(fm) == 0 || len(to) == 0 {
+ return 0
+ }
+
+ n := len(fm)
+ if len(to) < n {
+ n = len(to)
+ }
+
+ if raceenabled {
+ callerpc := gogetcallerpc(unsafe.Pointer(&to))
+ fn := slicestringcopy
+ pc := **(**uintptr)(unsafe.Pointer(&fn))
+ racewriterangepc(unsafe.Pointer(&to[0]), n, callerpc, pc)
+ }
+
+ memmove(unsafe.Pointer(&to[0]), unsafe.Pointer((*stringStruct)(unsafe.Pointer(&fm)).str), uintptr(n))
+ return n
+}
+
+var printslice_m byte
+
+func printslice(a sliceStruct) {
+ mp := acquirem()
+ mp.ptrarg[0] = a.array
+ mp.scalararg[0] = uint(a.len)
+ mp.scalararg[1] = uint(a.cap)
+ mcall(&printslice_m)
+ releasem(mp)
+}
diff --git a/src/pkg/runtime/slice.goc b/src/pkg/runtime/slice.goc
deleted file mode 100644
index 1b33ea5..0000000
--- a/src/pkg/runtime/slice.goc
+++ /dev/null
@@ -1,208 +0,0 @@
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package runtime
-#include "runtime.h"
-#include "arch_GOARCH.h"
-#include "type.h"
-#include "typekind.h"
-#include "malloc.h"
-#include "race.h"
-#include "stack.h"
-#include "../../cmd/ld/textflag.h"
-
-enum
-{
- debug = 0
-};
-
-static void makeslice1(SliceType*, intgo, intgo, Slice*);
-static void growslice1(SliceType*, Slice, intgo, Slice *);
-
-// see also unsafe·NewArray
-func makeslice(t *SliceType, len int64, cap int64) (ret Slice) {
- // NOTE: The len > MaxMem/elemsize check here is not strictly necessary,
- // but it produces a 'len out of range' error instead of a 'cap out of range' error
- // when someone does make([]T, bignumber). 'cap out of range' is true too,
- // but since the cap is only being supplied implicitly, saying len is clearer.
- // See issue 4085.
- if(len < 0 || (intgo)len != len || t->elem->size > 0 && len > MaxMem / t->elem->size)
- runtime·panicstring("makeslice: len out of range");
-
- if(cap < len || (intgo)cap != cap || t->elem->size > 0 && cap > MaxMem / t->elem->size)
- runtime·panicstring("makeslice: cap out of range");
-
- makeslice1(t, len, cap, &ret);
-
- if(debug) {
- runtime·printf("makeslice(%S, %D, %D); ret=",
- *t->string, len, cap);
- runtime·printslice(ret);
- }
-}
-
-// Dummy word to use as base pointer for make([]T, 0).
-// Since you cannot take the address of such a slice,
-// you can't tell that they all have the same base pointer.
-uintptr runtime·zerobase;
-
-static void
-makeslice1(SliceType *t, intgo len, intgo cap, Slice *ret)
-{
- ret->len = len;
- ret->cap = cap;
- ret->array = runtime·cnewarray(t->elem, cap);
-}
-
-// growslice(type *Type, x, []T, n int64) []T
-func growslice(t *SliceType, old Slice, n int64) (ret Slice) {
- int64 cap;
- void *pc;
-
- if(n < 1)
- runtime·panicstring("growslice: invalid n");
-
- cap = old.cap + n;
-
- if((intgo)cap != cap || cap < (int64)old.cap || (t->elem->size > 0 && cap > MaxMem/t->elem->size))
- runtime·panicstring("growslice: cap out of range");
-
- if(raceenabled) {
- pc = runtime·getcallerpc(&t);
- runtime·racereadrangepc(old.array, old.len*t->elem->size, pc, runtime·growslice);
- }
-
- growslice1(t, old, cap, &ret);
-
- if(debug) {
- runtime·printf("growslice(%S,", *t->string);
- runtime·printslice(old);
- runtime·printf(", new cap=%D) =", cap);
- runtime·printslice(ret);
- }
-}
-
-static void
-growslice1(SliceType *t, Slice x, intgo newcap, Slice *ret)
-{
- intgo newcap1;
- uintptr capmem, lenmem;
- int32 flag;
- Type *typ;
-
- typ = t->elem;
- if(typ->size == 0) {
- *ret = x;
- ret->cap = newcap;
- return;
- }
-
- newcap1 = x.cap;
-
- // Using newcap directly for m+m < newcap handles
- // both the case where m == 0 and also the case where
- // m+m/4 wraps around, in which case the loop
- // below might never terminate.
- if(newcap1+newcap1 < newcap)
- newcap1 = newcap;
- else {
- do {
- if(x.len < 1024)
- newcap1 += newcap1;
- else
- newcap1 += newcap1/4;
- } while(newcap1 < newcap);
- }
-
- if(newcap1 > MaxMem/typ->size)
- runtime·panicstring("growslice: cap out of range");
- // Try to use all memory that malloc will give us...
- capmem = runtime·roundupsize(newcap1*typ->size);
- // ...but don't ask for fractional number of elements (that can confuse GC).
- newcap1 = capmem/typ->size;
- capmem = newcap1*typ->size;
- flag = 0;
- // Can't use FlagNoZero w/o FlagNoScan, because otherwise GC can scan unitialized memory.
- if(typ->kind&KindNoPointers)
- flag = FlagNoScan|FlagNoZero;
- ret->array = runtime·mallocgc(capmem, typ, flag);
- ret->len = x.len;
- ret->cap = newcap1;
- lenmem = x.len*typ->size;
- runtime·memmove(ret->array, x.array, lenmem);
- if(typ->kind&KindNoPointers)
- runtime·memclr(ret->array+lenmem, capmem-lenmem);
-}
-
-#pragma textflag NOSPLIT
-func copy(to Slice, fm Slice, width uintptr) (ret int) {
- void *pc;
-
- if(fm.len == 0 || to.len == 0 || width == 0) {
- ret = 0;
- goto out;
- }
-
- ret = fm.len;
- if(to.len < ret)
- ret = to.len;
-
- if(raceenabled) {
- pc = runtime·getcallerpc(&to);
- runtime·racewriterangepc(to.array, ret*width, pc, runtime·copy);
- runtime·racereadrangepc(fm.array, ret*width, pc, runtime·copy);
- }
-
- if(ret == 1 && width == 1) { // common case worth about 2x to do here
- *to.array = *fm.array; // known to be a byte pointer
- } else {
- runtime·memmove(to.array, fm.array, ret*width);
- }
-
-out:
-
- if(debug) {
- runtime·prints("main·copy: to=");
- runtime·printslice(to);
- runtime·prints("; fm=");
- runtime·printslice(fm);
- runtime·prints("; width=");
- runtime·printint(width);
- runtime·prints("; ret=");
- runtime·printint(ret);
- runtime·prints("\n");
- }
-}
-
-#pragma textflag NOSPLIT
-func slicestringcopy(to Slice, fm String) (ret int) {
- void *pc;
-
- if(fm.len == 0 || to.len == 0) {
- ret = 0;
- goto out;
- }
-
- ret = fm.len;
- if(to.len < ret)
- ret = to.len;
-
- if(raceenabled) {
- pc = runtime·getcallerpc(&to);
- runtime·racewriterangepc(to.array, ret, pc, runtime·slicestringcopy);
- }
-
- runtime·memmove(to.array, fm.str, ret);
-
-out:;
-}
-
-func printslice(a Slice) {
- runtime·prints("[");
- runtime·printint(a.len);
- runtime·prints("/");
- runtime·printint(a.cap);
- runtime·prints("]");
- runtime·printpointer(a.array);
-}
diff --git a/src/pkg/runtime/stubs.go b/src/pkg/runtime/stubs.go
index ceb8031..fa1fa85 100644
--- a/src/pkg/runtime/stubs.go
+++ b/src/pkg/runtime/stubs.go
@@ -27,6 +27,9 @@
//go:noescape
func racereadrangepc(addr unsafe.Pointer, len int, callpc, pc uintptr)
+//go:noescape
+func racewriterangepc(addr unsafe.Pointer, len int, callpc, pc uintptr)
+
// Should be a built-in for unsafe.Pointer?
func add(p unsafe.Pointer, x uintptr) unsafe.Pointer {
return unsafe.Pointer(uintptr(p) + x)
