| // Copyright 2014 The Go Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style | |
| // license that can be found in the LICENSE file. | |
| package runtime | |
| import ( | |
| "internal/abi" | |
| "internal/goarch" | |
| "internal/runtime/atomic" | |
| "internal/runtime/sys" | |
| "unsafe" | |
| ) | |
| const itabInitSize = 512 | |
| var ( | |
| itabLock mutex // lock for accessing itab table | |
| itabTable = &itabTableInit // pointer to current table | |
| itabTableInit = itabTableType{size: itabInitSize} // starter table | |
| ) | |
| // Note: change the formula in the mallocgc call in itabAdd if you change these fields. | |
| type itabTableType struct { | |
| size uintptr // length of entries array. Always a power of 2. | |
| count uintptr // current number of filled entries. | |
| entries [itabInitSize]*itab // really [size] large | |
| } | |
| func itabHashFunc(inter *interfacetype, typ *_type) uintptr { | |
| // compiler has provided some good hash codes for us. | |
| return uintptr(inter.Type.Hash ^ typ.Hash) | |
| } | |
| // getitab should be an internal detail, | |
| // but widely used packages access it using linkname. | |
| // Notable members of the hall of shame include: | |
| // - github.com/bytedance/sonic | |
| // | |
| // Do not remove or change the type signature. | |
| // See go.dev/issue/67401. | |
| // | |
| //go:linkname getitab | |
| func getitab(inter *interfacetype, typ *_type, canfail bool) *itab { | |
| if len(inter.Methods) == 0 { | |
| throw("internal error - misuse of itab") | |
| } | |
| // easy case | |
| if typ.TFlag&abi.TFlagUncommon == 0 { | |
| if canfail { | |
| return nil | |
| } | |
| name := toRType(&inter.Type).nameOff(inter.Methods[0].Name) | |
| panic(&TypeAssertionError{nil, typ, &inter.Type, name.Name()}) | |
| } | |
| var m *itab | |
| // First, look in the existing table to see if we can find the itab we need. | |
| // This is by far the most common case, so do it without locks. | |
| // Use atomic to ensure we see any previous writes done by the thread | |
| // that updates the itabTable field (with atomic.Storep in itabAdd). | |
| t := (*itabTableType)(atomic.Loadp(unsafe.Pointer(&itabTable))) | |
| if m = t.find(inter, typ); m != nil { | |
| goto finish | |
| } | |
| // Not found. Grab the lock and try again. | |
| lock(&itabLock) | |
| if m = itabTable.find(inter, typ); m != nil { | |
| unlock(&itabLock) | |
| goto finish | |
| } | |
| // Entry doesn't exist yet. Make a new entry & add it. | |
| m = (*itab)(persistentalloc(unsafe.Sizeof(itab{})+uintptr(len(inter.Methods)-1)*goarch.PtrSize, 0, &memstats.other_sys)) | |
| m.Inter = inter | |
| m.Type = typ | |
| // The hash is used in type switches. However, compiler statically generates itab's | |
| // for all interface/type pairs used in switches (which are added to itabTable | |
| // in itabsinit). The dynamically-generated itab's never participate in type switches, | |
| // and thus the hash is irrelevant. | |
| // Note: m.Hash is _not_ the hash used for the runtime itabTable hash table. | |
| m.Hash = 0 | |
| itabInit(m, true) | |
| itabAdd(m) | |
| unlock(&itabLock) | |
| finish: | |
| if m.Fun[0] != 0 { | |
| return m | |
| } | |
| if canfail { | |
| return nil | |
| } | |
| // this can only happen if the conversion | |
| // was already done once using the , ok form | |
| // and we have a cached negative result. | |
| // The cached result doesn't record which | |
| // interface function was missing, so initialize | |
| // the itab again to get the missing function name. | |
| panic(&TypeAssertionError{concrete: typ, asserted: &inter.Type, missingMethod: itabInit(m, false)}) | |
| } | |
| // find finds the given interface/type pair in t. | |
| // Returns nil if the given interface/type pair isn't present. | |
| func (t *itabTableType) find(inter *interfacetype, typ *_type) *itab { | |
| // Implemented using quadratic probing. | |
| // Probe sequence is h(i) = h0 + i*(i+1)/2 mod 2^k. | |
| // We're guaranteed to hit all table entries using this probe sequence. | |
| mask := t.size - 1 | |
| h := itabHashFunc(inter, typ) & mask | |
| for i := uintptr(1); ; i++ { | |
| p := (**itab)(add(unsafe.Pointer(&t.entries), h*goarch.PtrSize)) | |
| // Use atomic read here so if we see m != nil, we also see | |
| // the initializations of the fields of m. | |
| // m := *p | |
| m := (*itab)(atomic.Loadp(unsafe.Pointer(p))) | |
| if m == nil { | |
| return nil | |
| } | |
| if m.Inter == inter && m.Type == typ { | |
| return m | |
| } | |
| h += i | |
| h &= mask | |
| } | |
| } | |
| // itabAdd adds the given itab to the itab hash table. | |
| // itabLock must be held. | |
| func itabAdd(m *itab) { | |
| // Bugs can lead to calling this while mallocing is set, | |
| // typically because this is called while panicking. | |
| // Crash reliably, rather than only when we need to grow | |
| // the hash table. | |
| if getg().m.mallocing != 0 { | |
| throw("malloc deadlock") | |
| } | |
| t := itabTable | |
| if t.count >= 3*(t.size/4) { // 75% load factor | |
| // Grow hash table. | |
| // t2 = new(itabTableType) + some additional entries | |
| // We lie and tell malloc we want pointer-free memory because | |
| // all the pointed-to values are not in the heap. | |
| t2 := (*itabTableType)(mallocgc((2+2*t.size)*goarch.PtrSize, nil, true)) | |
| t2.size = t.size * 2 | |
| // Copy over entries. | |
| // Note: while copying, other threads may look for an itab and | |
| // fail to find it. That's ok, they will then try to get the itab lock | |
| // and as a consequence wait until this copying is complete. | |
| iterate_itabs(t2.add) | |
| if t2.count != t.count { | |
| throw("mismatched count during itab table copy") | |
| } | |
| // Publish new hash table. Use an atomic write: see comment in getitab. | |
| atomicstorep(unsafe.Pointer(&itabTable), unsafe.Pointer(t2)) | |
| // Adopt the new table as our own. | |
| t = itabTable | |
| // Note: the old table can be GC'ed here. | |
| } | |
| t.add(m) | |
| } | |
| // add adds the given itab to itab table t. | |
| // itabLock must be held. | |
| func (t *itabTableType) add(m *itab) { | |
| // See comment in find about the probe sequence. | |
| // Insert new itab in the first empty spot in the probe sequence. | |
| mask := t.size - 1 | |
| h := itabHashFunc(m.Inter, m.Type) & mask | |
| for i := uintptr(1); ; i++ { | |
| p := (**itab)(add(unsafe.Pointer(&t.entries), h*goarch.PtrSize)) | |
| m2 := *p | |
| if m2 != nil && m2.Inter == m.Inter && m2.Type == m.Type { | |
| // A plugin has its own copy of the itabs that the main program | |
| // also has. Type switches and type assertions compare against | |
| // the itab that is already in the table, so don't add a second | |
| // itab for the same interface/type pair. | |
| return | |
| } | |
| if m2 == nil { | |
| // Use atomic write here so if a reader sees m, it also | |
| // sees the correctly initialized fields of m. | |
| // NoWB is ok because m is not in heap memory. | |
| // *p = m | |
| atomic.StorepNoWB(unsafe.Pointer(p), unsafe.Pointer(m)) | |
| t.count++ | |
| return | |
| } | |
| h += i | |
| h &= mask | |
| } | |
| } | |
| // itabInit fills in the m.Fun array with all the code pointers for | |
| // the m.Inter/m.Type pair. If the type does not implement the interface, | |
| // it sets m.Fun[0] to 0 and returns the name of an interface function that is missing. | |
| // If !firstTime, itabInit will not write anything to m.Fun (see issue 65962). | |
| // It is ok to call this multiple times on the same m, even concurrently | |
| // (although it will only be called once with firstTime==true). | |
| // | |
| // itabInit stores only code pointers, so it must not emit a write barrier; | |
| // //go:nowritebarrier enforces that (see the methods store below). | |
| // | |
| //go:nowritebarrier | |
| func itabInit(m *itab, firstTime bool) string { | |
| inter := m.Inter | |
| typ := m.Type | |
| x := typ.Uncommon() | |
| // both inter and typ have method sorted by name, | |
| // and interface names are unique, | |
| // so can iterate over both in lock step; | |
| // the loop is O(ni+nt) not O(ni*nt). | |
| ni := len(inter.Methods) | |
| nt := int(x.Mcount) | |
| xmhdr := unsafe.Slice((*abi.Method)(add(unsafe.Pointer(x), uintptr(x.Moff))), nt) | |
| j := 0 | |
| // These are code pointers, not heap pointers. Use a uintptr slice so the | |
| // store emits no write barrier: on wasm a code pointer can look like a | |
| // heap pointer and make the GC crash (see issue 80472). | |
| methods := unsafe.Slice(&m.Fun[0], ni) | |
| var fun0 unsafe.Pointer | |
| imethods: | |
| for k := 0; k < ni; k++ { | |
| i := &inter.Methods[k] | |
| itype := toRType(&inter.Type).typeOff(i.Typ) | |
| name := toRType(&inter.Type).nameOff(i.Name) | |
| iname := name.Name() | |
| ipkg := pkgPath(name) | |
| if ipkg == "" { | |
| ipkg = inter.PkgPath.Name() | |
| } | |
| for ; j < nt; j++ { | |
| t := &xmhdr[j] | |
| rtyp := toRType(typ) | |
| tname := rtyp.nameOff(t.Name) | |
| if rtyp.typeOff(t.Mtyp) == itype && tname.Name() == iname { | |
| pkgPath := pkgPath(tname) | |
| if pkgPath == "" { | |
| pkgPath = rtyp.nameOff(x.PkgPath).Name() | |
| } | |
| if tname.IsExported() || pkgPath == ipkg { | |
| ifn := rtyp.textOff(t.Ifn) | |
| if k == 0 { | |
| fun0 = ifn // we'll set m.Fun[0] at the end | |
| } else if firstTime { | |
| methods[k] = uintptr(ifn) | |
| } | |
| continue imethods | |
| } | |
| } | |
| } | |
| // didn't find method | |
| // Leaves m.Fun[0] set to 0. | |
| return iname | |
| } | |
| if firstTime { | |
| m.Fun[0] = uintptr(fun0) | |
| } | |
| return "" | |
| } | |
| func itabsinit() { | |
| lockInit(&itabLock, lockRankItab) | |
| lock(&itabLock) | |
| for _, md := range activeModules() { | |
| addModuleItabs(md) | |
| } | |
| unlock(&itabLock) | |
| } | |
| // addModuleItabs adds the pre-compiled itabs from md to the itab hash table. | |
| // This is an optimization to let us skip creating itabs we already have. | |
| func addModuleItabs(md *moduledata) { | |
| p := md.types + md.itaboffset | |
| end := p + md.itabsize | |
| for p < end { | |
| itab := (*itab)(unsafe.Pointer(p)) | |
| itabAdd(itab) | |
| p += uintptr(itab.Size()) | |
| } | |
| } | |
| // panicdottypeE is called when doing an e.(T) conversion and the conversion fails. | |
| // have = the dynamic type we have. | |
| // want = the static type we're trying to convert to. | |
| // iface = the static type we're converting from. | |
| func panicdottypeE(have, want, iface *_type) { | |
| panic(&TypeAssertionError{iface, have, want, ""}) | |
| } | |
| // panicdottypeI is called when doing an i.(T) conversion and the conversion fails. | |
| // Same args as panicdottypeE, but "have" is the dynamic itab we have. | |
| func panicdottypeI(have *itab, want, iface *_type) { | |
| var t *_type | |
| if have != nil { | |
| t = have.Type | |
| } | |
| panicdottypeE(t, want, iface) | |
| } | |
| // panicnildottype is called when doing an i.(T) conversion and the interface i is nil. | |
| // want = the static type we're trying to convert to. | |
| func panicnildottype(want *_type) { | |
| panic(&TypeAssertionError{nil, nil, want, ""}) | |
| // TODO: Add the static type we're converting from as well. | |
| // It might generate a better error message. | |
| // Just to match other nil conversion errors, we don't for now. | |
| } | |
| // The specialized convTx routines need a type descriptor to use when calling mallocgc. | |
| // We don't need the type to be exact, just to have the correct size, alignment, and pointer-ness. | |
| // However, when debugging, it'd be nice to have some indication in mallocgc where the types came from, | |
| // so we use named types here. | |
| // We then construct interface values of these types, | |
| // and then extract the type word to use as needed. | |
| type ( | |
| uint16InterfacePtr uint16 | |
| uint32InterfacePtr uint32 | |
| uint64InterfacePtr uint64 | |
| stringInterfacePtr string | |
| sliceInterfacePtr []byte | |
| ) | |
| var ( | |
| uint16Eface any = uint16InterfacePtr(0) | |
| uint32Eface any = uint32InterfacePtr(0) | |
| uint64Eface any = uint64InterfacePtr(0) | |
| stringEface any = stringInterfacePtr("") | |
| sliceEface any = sliceInterfacePtr(nil) | |
| uint16Type *_type = efaceOf(&uint16Eface)._type | |
| uint32Type *_type = efaceOf(&uint32Eface)._type | |
| uint64Type *_type = efaceOf(&uint64Eface)._type | |
| stringType *_type = efaceOf(&stringEface)._type | |
| sliceType *_type = efaceOf(&sliceEface)._type | |
| ) | |
| // The conv and assert functions below do very similar things. | |
| // The convXXX functions are guaranteed by the compiler to succeed. | |
| // The assertXXX functions may fail (either panicking or returning false, | |
| // depending on whether they are 1-result or 2-result). | |
| // The convXXX functions succeed on a nil input, whereas the assertXXX | |
| // functions fail on a nil input. | |
| // convT converts a value of type t, which is pointed to by v, to a pointer that can | |
| // be used as the second word of an interface value. | |
| func convT(t *_type, v unsafe.Pointer) unsafe.Pointer { | |
| if raceenabled { | |
| raceReadObjectPC(t, v, sys.GetCallerPC(), abi.FuncPCABIInternal(convT)) | |
| } | |
| if msanenabled { | |
| msanread(v, t.Size_) | |
| } | |
| if asanenabled { | |
| asanread(v, t.Size_) | |
| } | |
| x := mallocgc(t.Size_, t, true) | |
| typedmemmove(t, x, v) | |
| return x | |
| } | |
| func convTnoptr(t *_type, v unsafe.Pointer) unsafe.Pointer { | |
| // TODO: maybe take size instead of type? | |
| if raceenabled { | |
| raceReadObjectPC(t, v, sys.GetCallerPC(), abi.FuncPCABIInternal(convTnoptr)) | |
| } | |
| if msanenabled { | |
| msanread(v, t.Size_) | |
| } | |
| if asanenabled { | |
| asanread(v, t.Size_) | |
| } | |
| x := mallocgc(t.Size_, t, false) | |
| memmove(x, v, t.Size_) | |
| return x | |
| } | |
| func convT16(val uint16) (x unsafe.Pointer) { | |
| if val < uint16(len(staticuint64s)) { | |
| x = unsafe.Pointer(&staticuint64s[val]) | |
| if goarch.BigEndian { | |
| x = add(x, 6) | |
| } | |
| } else { | |
| x = mallocgc(2, uint16Type, false) | |
| *(*uint16)(x) = val | |
| } | |
| return | |
| } | |
| func convT32(val uint32) (x unsafe.Pointer) { | |
| if val < uint32(len(staticuint64s)) { | |
| x = unsafe.Pointer(&staticuint64s[val]) | |
| if goarch.BigEndian { | |
| x = add(x, 4) | |
| } | |
| } else { | |
| x = mallocgc(4, uint32Type, false) | |
| *(*uint32)(x) = val | |
| } | |
| return | |
| } | |
| // convT64 should be an internal detail, | |
| // but widely used packages access it using linkname. | |
| // Notable members of the hall of shame include: | |
| // - github.com/bytedance/sonic | |
| // | |
| // Do not remove or change the type signature. | |
| // See go.dev/issue/67401. | |
| // | |
| //go:linkname convT64 | |
| func convT64(val uint64) (x unsafe.Pointer) { | |
| if val < uint64(len(staticuint64s)) { | |
| x = unsafe.Pointer(&staticuint64s[val]) | |
| } else { | |
| x = mallocgc(8, uint64Type, false) | |
| *(*uint64)(x) = val | |
| } | |
| return | |
| } | |
| // convTstring should be an internal detail, | |
| // but widely used packages access it using linkname. | |
| // Notable members of the hall of shame include: | |
| // - github.com/bytedance/sonic | |
| // | |
| // Do not remove or change the type signature. | |
| // See go.dev/issue/67401. | |
| // | |
| //go:linkname convTstring | |
| func convTstring(val string) (x unsafe.Pointer) { | |
| if val == "" { | |
| x = unsafe.Pointer(&zeroVal[0]) | |
| } else { | |
| x = mallocgc(unsafe.Sizeof(val), stringType, true) | |
| *(*string)(x) = val | |
| } | |
| return | |
| } | |
| // convTslice should be an internal detail, | |
| // but widely used packages access it using linkname. | |
| // Notable members of the hall of shame include: | |
| // - github.com/bytedance/sonic | |
| // | |
| // Do not remove or change the type signature. | |
| // See go.dev/issue/67401. | |
| // | |
| //go:linkname convTslice | |
| func convTslice(val []byte) (x unsafe.Pointer) { | |
| // Note: this must work for any element type, not just byte. | |
| if (*slice)(unsafe.Pointer(&val)).array == nil { | |
| x = unsafe.Pointer(&zeroVal[0]) | |
| } else { | |
| x = mallocgc(unsafe.Sizeof(val), sliceType, true) | |
| *(*[]byte)(x) = val | |
| } | |
| return | |
| } | |
| func assertE2I(inter *interfacetype, t *_type) *itab { | |
| if t == nil { | |
| // explicit conversions require non-nil interface value. | |
| panic(&TypeAssertionError{nil, nil, &inter.Type, ""}) | |
| } | |
| return getitab(inter, t, false) | |
| } | |
| func assertE2I2(inter *interfacetype, t *_type) *itab { | |
| if t == nil { | |
| return nil | |
| } | |
| return getitab(inter, t, true) | |
| } | |
| // typeAssert builds an itab for the concrete type t and the | |
| // interface type s.Inter. If the conversion is not possible it | |
| // panics if s.CanFail is false and returns nil if s.CanFail is true. | |
| func typeAssert(s *abi.TypeAssert, t *_type) *itab { | |
| var tab *itab | |
| if t == nil { | |
| if !s.CanFail { | |
| panic(&TypeAssertionError{nil, nil, &s.Inter.Type, ""}) | |
| } | |
| } else { | |
| tab = getitab(s.Inter, t, s.CanFail) | |
| } | |
| if !abi.UseInterfaceSwitchCache(goarch.ArchFamily) { | |
| return tab | |
| } | |
| // Maybe update the cache, so the next time the generated code | |
| // doesn't need to call into the runtime. | |
| if cheaprand()&1023 != 0 { | |
| // Only bother updating the cache ~1 in 1000 times. | |
| return tab | |
| } | |
| // Load the current cache. | |
| oldC := (*abi.TypeAssertCache)(atomic.Loadp(unsafe.Pointer(&s.Cache))) | |
| if cheaprand()&uint32(oldC.Mask) != 0 { | |
| // As cache gets larger, choose to update it less often | |
| // so we can amortize the cost of building a new cache. | |
| return tab | |
| } | |
| // Make a new cache. | |
| newC := buildTypeAssertCache(oldC, t, tab) | |
| // Update cache. Use compare-and-swap so if multiple threads | |
| // are fighting to update the cache, at least one of their | |
| // updates will stick. | |
| atomic_casPointer((*unsafe.Pointer)(unsafe.Pointer(&s.Cache)), unsafe.Pointer(oldC), unsafe.Pointer(newC)) | |
| return tab | |
| } | |
| func buildTypeAssertCache(oldC *abi.TypeAssertCache, typ *_type, tab *itab) *abi.TypeAssertCache { | |
| oldEntries := unsafe.Slice(&oldC.Entries[0], oldC.Mask+1) | |
| // Count the number of entries we need. | |
| n := 1 | |
| for _, e := range oldEntries { | |
| if e.Typ != 0 { | |
| n++ | |
| } | |
| } | |
| // Figure out how big a table we need. | |
| // We need at least one more slot than the number of entries | |
| // so that we are guaranteed an empty slot (for termination). | |
| newN := n * 2 // make it at most 50% full | |
| newN = 1 << sys.Len64(uint64(newN-1)) // round up to a power of 2 | |
| // Allocate the new table. | |
| newSize := unsafe.Sizeof(abi.TypeAssertCache{}) + uintptr(newN-1)*unsafe.Sizeof(abi.TypeAssertCacheEntry{}) | |
| newC := (*abi.TypeAssertCache)(mallocgc(newSize, nil, true)) | |
| newC.Mask = uintptr(newN - 1) | |
| newEntries := unsafe.Slice(&newC.Entries[0], newN) | |
| // Fill the new table. | |
| addEntry := func(typ *_type, tab *itab) { | |
| h := int(typ.Hash) & (newN - 1) | |
| for { | |
| if newEntries[h].Typ == 0 { | |
| newEntries[h].Typ = uintptr(unsafe.Pointer(typ)) | |
| newEntries[h].Itab = uintptr(unsafe.Pointer(tab)) | |
| return | |
| } | |
| h = (h + 1) & (newN - 1) | |
| } | |
| } | |
| for _, e := range oldEntries { | |
| if e.Typ != 0 { | |
| addEntry((*_type)(unsafe.Pointer(e.Typ)), (*itab)(unsafe.Pointer(e.Itab))) | |
| } | |
| } | |
| addEntry(typ, tab) | |
| return newC | |
| } | |
| // Empty type assert cache. Contains one entry with a nil Typ (which | |
| // causes a cache lookup to fail immediately.) | |
| var emptyTypeAssertCache = abi.TypeAssertCache{Mask: 0} | |
| // interfaceSwitch compares t against the list of cases in s. | |
| // If t matches case i, interfaceSwitch returns the case index i and | |
| // an itab for the pair <t, s.Cases[i]>. | |
| // If there is no match, return N,nil, where N is the number | |
| // of cases. | |
| func interfaceSwitch(s *abi.InterfaceSwitch, t *_type) (int, *itab) { | |
| cases := unsafe.Slice(&s.Cases[0], s.NCases) | |
| // Results if we don't find a match. | |
| case_ := len(cases) | |
| var tab *itab | |
| // Look through each case in order. | |
| for i, c := range cases { | |
| tab = getitab(c, t, true) | |
| if tab != nil { | |
| case_ = i | |
| break | |
| } | |
| } | |
| if !abi.UseInterfaceSwitchCache(goarch.ArchFamily) { | |
| return case_, tab | |
| } | |
| // Maybe update the cache, so the next time the generated code | |
| // doesn't need to call into the runtime. | |
| if cheaprand()&1023 != 0 { | |
| // Only bother updating the cache ~1 in 1000 times. | |
| // This ensures we don't waste memory on switches, or | |
| // switch arguments, that only happen a few times. | |
| return case_, tab | |
| } | |
| // Load the current cache. | |
| oldC := (*abi.InterfaceSwitchCache)(atomic.Loadp(unsafe.Pointer(&s.Cache))) | |
| if cheaprand()&uint32(oldC.Mask) != 0 { | |
| // As cache gets larger, choose to update it less often | |
| // so we can amortize the cost of building a new cache | |
| // (that cost is linear in oldc.Mask). | |
| return case_, tab | |
| } | |
| // Make a new cache. | |
| newC := buildInterfaceSwitchCache(oldC, t, case_, tab) | |
| // Update cache. Use compare-and-swap so if multiple threads | |
| // are fighting to update the cache, at least one of their | |
| // updates will stick. | |
| atomic_casPointer((*unsafe.Pointer)(unsafe.Pointer(&s.Cache)), unsafe.Pointer(oldC), unsafe.Pointer(newC)) | |
| return case_, tab | |
| } | |
| // buildInterfaceSwitchCache constructs an interface switch cache | |
| // containing all the entries from oldC plus the new entry | |
| // (typ,case_,tab). | |
| func buildInterfaceSwitchCache(oldC *abi.InterfaceSwitchCache, typ *_type, case_ int, tab *itab) *abi.InterfaceSwitchCache { | |
| oldEntries := unsafe.Slice(&oldC.Entries[0], oldC.Mask+1) | |
| // Count the number of entries we need. | |
| n := 1 | |
| for _, e := range oldEntries { | |
| if e.Typ != 0 { | |
| n++ | |
| } | |
| } | |
| // Figure out how big a table we need. | |
| // We need at least one more slot than the number of entries | |
| // so that we are guaranteed an empty slot (for termination). | |
| newN := n * 2 // make it at most 50% full | |
| newN = 1 << sys.Len64(uint64(newN-1)) // round up to a power of 2 | |
| // Allocate the new table. | |
| newSize := unsafe.Sizeof(abi.InterfaceSwitchCache{}) + uintptr(newN-1)*unsafe.Sizeof(abi.InterfaceSwitchCacheEntry{}) | |
| newC := (*abi.InterfaceSwitchCache)(mallocgc(newSize, nil, true)) | |
| newC.Mask = uintptr(newN - 1) | |
| newEntries := unsafe.Slice(&newC.Entries[0], newN) | |
| // Fill the new table. | |
| addEntry := func(typ *_type, case_ int, tab *itab) { | |
| h := int(typ.Hash) & (newN - 1) | |
| for { | |
| if newEntries[h].Typ == 0 { | |
| newEntries[h].Typ = uintptr(unsafe.Pointer(typ)) | |
| newEntries[h].Case = case_ | |
| newEntries[h].Itab = uintptr(unsafe.Pointer(tab)) | |
| return | |
| } | |
| h = (h + 1) & (newN - 1) | |
| } | |
| } | |
| for _, e := range oldEntries { | |
| if e.Typ != 0 { | |
| addEntry((*_type)(unsafe.Pointer(e.Typ)), e.Case, (*itab)(unsafe.Pointer(e.Itab))) | |
| } | |
| } | |
| addEntry(typ, case_, tab) | |
| return newC | |
| } | |
| // Empty interface switch cache. Contains one entry with a nil Typ (which | |
| // causes a cache lookup to fail immediately.) | |
| var emptyInterfaceSwitchCache = abi.InterfaceSwitchCache{Mask: 0} | |
| // reflect_ifaceE2I is for package reflect, | |
| // but widely used packages access it using linkname. | |
| // Notable members of the hall of shame include: | |
| // - gitee.com/quant1x/gox | |
| // - github.com/modern-go/reflect2 | |
| // - github.com/v2pro/plz | |
| // | |
| // Do not remove or change the type signature. | |
| // | |
| //go:linkname reflect_ifaceE2I reflect.ifaceE2I | |
| func reflect_ifaceE2I(inter *interfacetype, e eface, dst *iface) { | |
| *dst = iface{assertE2I(inter, e._type), e.data} | |
| } | |
| //go:linkname reflectlite_ifaceE2I internal/reflectlite.ifaceE2I | |
| func reflectlite_ifaceE2I(inter *interfacetype, e eface, dst *iface) { | |
| *dst = iface{assertE2I(inter, e._type), e.data} | |
| } | |
| func iterate_itabs(fn func(*itab)) { | |
| // Note: only runs during stop the world or with itabLock held, | |
| // so no other locks/atomics needed. | |
| t := itabTable | |
| for i := uintptr(0); i < t.size; i++ { | |
| m := *(**itab)(add(unsafe.Pointer(&t.entries), i*goarch.PtrSize)) | |
| if m != nil { | |
| fn(m) | |
| } | |
| } | |
| } | |
| // staticuint64s is used to avoid allocating in convTx for small integer values. | |
| // staticuint64s[0] == 0, staticuint64s[1] == 1, and so forth. | |
| // It is defined in assembler code so that it is read-only. | |
| var staticuint64s [256]uint64 | |
| // getStaticuint64s is called by the reflect package to get a pointer | |
| // to the read-only array. | |
| // | |
| //go:linkname getStaticuint64s | |
| func getStaticuint64s() *[256]uint64 { | |
| return &staticuint64s | |
| } | |
| // The linker redirects a reference of a method that it determined | |
| // unreachable to a reference to this function, so it will throw if | |
| // ever called. | |
| func unreachableMethod() { | |
| throw("unreachable method called. linker bug?") | |
| } |