| // Copyright 2018 The Go Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style | |
| // license that can be found in the LICENSE file. | |
| //go:build 386 || amd64 || amd64p32 | |
| package cpu | |
| import "runtime" | |
| const cacheLineSize = 64 | |
| func initOptions() { | |
| options = []option{ | |
| {Name: "adx", Feature: &X86.HasADX}, | |
| {Name: "aes", Feature: &X86.HasAES}, | |
| {Name: "avx", Feature: &X86.HasAVX}, | |
| {Name: "avx2", Feature: &X86.HasAVX2}, | |
| {Name: "avx512", Feature: &X86.HasAVX512}, | |
| {Name: "avx512f", Feature: &X86.HasAVX512F}, | |
| {Name: "avx512cd", Feature: &X86.HasAVX512CD}, | |
| {Name: "avx512er", Feature: &X86.HasAVX512ER}, | |
| {Name: "avx512pf", Feature: &X86.HasAVX512PF}, | |
| {Name: "avx512vl", Feature: &X86.HasAVX512VL}, | |
| {Name: "avx512bw", Feature: &X86.HasAVX512BW}, | |
| {Name: "avx512dq", Feature: &X86.HasAVX512DQ}, | |
| {Name: "avx512ifma", Feature: &X86.HasAVX512IFMA}, | |
| {Name: "avx512vbmi", Feature: &X86.HasAVX512VBMI}, | |
| {Name: "avx512vnniw", Feature: &X86.HasAVX5124VNNIW}, | |
| {Name: "avx5124fmaps", Feature: &X86.HasAVX5124FMAPS}, | |
| {Name: "avx512vpopcntdq", Feature: &X86.HasAVX512VPOPCNTDQ}, | |
| {Name: "avx512vpclmulqdq", Feature: &X86.HasAVX512VPCLMULQDQ}, | |
| {Name: "avx512vnni", Feature: &X86.HasAVX512VNNI}, | |
| {Name: "avx512gfni", Feature: &X86.HasAVX512GFNI}, | |
| {Name: "avx512vaes", Feature: &X86.HasAVX512VAES}, | |
| {Name: "avx512vbmi2", Feature: &X86.HasAVX512VBMI2}, | |
| {Name: "avx512bitalg", Feature: &X86.HasAVX512BITALG}, | |
| {Name: "avx512bf16", Feature: &X86.HasAVX512BF16}, | |
| {Name: "amxtile", Feature: &X86.HasAMXTile}, | |
| {Name: "amxint8", Feature: &X86.HasAMXInt8}, | |
| {Name: "amxbf16", Feature: &X86.HasAMXBF16}, | |
| {Name: "bmi1", Feature: &X86.HasBMI1}, | |
| {Name: "bmi2", Feature: &X86.HasBMI2}, | |
| {Name: "cx16", Feature: &X86.HasCX16}, | |
| {Name: "erms", Feature: &X86.HasERMS}, | |
| {Name: "f16c", Feature: &X86.HasF16C}, | |
| {Name: "fma", Feature: &X86.HasFMA}, | |
| {Name: "lahf", Feature: &X86.HasLAHF}, | |
| {Name: "lzcnt", Feature: &X86.HasLZCNT}, | |
| {Name: "movbe", Feature: &X86.HasMOVBE}, | |
| {Name: "osxsave", Feature: &X86.HasOSXSAVE}, | |
| {Name: "pclmulqdq", Feature: &X86.HasPCLMULQDQ}, | |
| {Name: "popcnt", Feature: &X86.HasPOPCNT}, | |
| {Name: "rdrand", Feature: &X86.HasRDRAND}, | |
| {Name: "rdseed", Feature: &X86.HasRDSEED}, | |
| {Name: "sse3", Feature: &X86.HasSSE3}, | |
| {Name: "sse41", Feature: &X86.HasSSE41}, | |
| {Name: "sse42", Feature: &X86.HasSSE42}, | |
| {Name: "ssse3", Feature: &X86.HasSSSE3}, | |
| {Name: "avxifma", Feature: &X86.HasAVXIFMA}, | |
| {Name: "avxvnni", Feature: &X86.HasAVXVNNI}, | |
| {Name: "avxvnniint8", Feature: &X86.HasAVXVNNIInt8}, | |
| {Name: "xsave", Feature: &X86.HasXSAVE}, | |
| // These capabilities should always be enabled on amd64: | |
| {Name: "sse2", Feature: &X86.HasSSE2, Required: runtime.GOARCH == "amd64"}, | |
| } | |
| } | |
| func archInit() { | |
| // From internal/cpu | |
| const ( | |
| // eax bits | |
| cpuid_AVXVNNI = 1 << 4 | |
| // ecx bits | |
| cpuid_SSE3 = 1 << 0 | |
| cpuid_PCLMULQDQ = 1 << 1 | |
| cpuid_AVX512VBMI = 1 << 1 | |
| cpuid_AVX512VBMI2 = 1 << 6 | |
| cpuid_SSSE3 = 1 << 9 | |
| cpuid_AVX512GFNI = 1 << 8 | |
| cpuid_AVX512VAES = 1 << 9 | |
| cpuid_AVX512VNNI = 1 << 11 | |
| cpuid_AVX512BITALG = 1 << 12 | |
| cpuid_FMA = 1 << 12 | |
| cpuid_AVX512VPOPCNTDQ = 1 << 14 | |
| cpuid_SSE41 = 1 << 19 | |
| cpuid_SSE42 = 1 << 20 | |
| cpuid_MOVBE = 1 << 22 | |
| cpuid_POPCNT = 1 << 23 | |
| cpuid_AES = 1 << 25 | |
| cpuid_XSAVE = 1 << 26 | |
| cpuid_OSXSAVE = 1 << 27 | |
| cpuid_AVX = 1 << 28 | |
| cpuid_F16C = 1 << 29 | |
| // "Extended Feature Flag" bits returned in EBX for CPUID EAX=0x7 ECX=0x0 | |
| cpuid_BMI1 = 1 << 3 | |
| cpuid_AVX2 = 1 << 5 | |
| cpuid_BMI2 = 1 << 8 | |
| cpuid_ERMS = 1 << 9 | |
| cpuid_AVX512F = 1 << 16 | |
| cpuid_AVX512DQ = 1 << 17 | |
| cpuid_ADX = 1 << 19 | |
| cpuid_AVX512CD = 1 << 28 | |
| cpuid_SHA = 1 << 29 | |
| cpuid_AVX512BW = 1 << 30 | |
| cpuid_AVX512VL = 1 << 31 | |
| // "Extended Feature Flag" bits returned in ECX for CPUID EAX=0x7 ECX=0x0 | |
| cpuid_AVX512_VBMI = 1 << 1 | |
| cpuid_AVX512_VBMI2 = 1 << 6 | |
| cpuid_GFNI = 1 << 8 | |
| cpuid_AVX512VPCLMULQDQ = 1 << 10 | |
| cpuid_AVX512_BITALG = 1 << 12 | |
| // edx bits | |
| cpuid_FSRM = 1 << 4 | |
| // ecx bits for CPUID 0x80000001 | |
| cpuid_LAHF = 1 << 0 | |
| cpuid_LZCNT = 1 << 5 | |
| // edx bits for CPUID 0x80000001 | |
| cpuid_RDTSCP = 1 << 27 | |
| ) | |
| // Additional constants not in internal/cpu | |
| const ( | |
| // eax=1: edx | |
| cpuid_SSE2 = 1 << 26 | |
| // eax=1: ecx | |
| cpuid_CX16 = 1 << 13 | |
| cpuid_RDRAND = 1 << 30 | |
| // eax=7,ecx=0: ebx | |
| cpuid_RDSEED = 1 << 18 | |
| cpuid_AVX512IFMA = 1 << 21 | |
| cpuid_AVX512PF = 1 << 26 | |
| cpuid_AVX512ER = 1 << 27 | |
| // eax=7,ecx=0: edx | |
| cpuid_AVX5124VNNIW = 1 << 2 | |
| cpuid_AVX5124FMAPS = 1 << 3 | |
| cpuid_AMXBF16 = 1 << 22 | |
| cpuid_AMXTile = 1 << 24 | |
| cpuid_AMXInt8 = 1 << 25 | |
| // eax=7,ecx=1: eax | |
| cpuid_AVX512BF16 = 1 << 5 | |
| cpuid_AVXIFMA = 1 << 23 | |
| // eax=7,ecx=1: edx | |
| cpuid_AVXVNNIInt8 = 1 << 4 | |
| ) | |
| Initialized = true | |
| maxID, _, _, _ := cpuid(0, 0) | |
| if maxID < 1 { | |
| return | |
| } | |
| _, _, ecx1, edx1 := cpuid(1, 0) | |
| X86.HasSSE2 = isSet(edx1, cpuid_SSE2) | |
| X86.HasSSE3 = isSet(ecx1, cpuid_SSE3) | |
| X86.HasPCLMULQDQ = isSet(ecx1, cpuid_PCLMULQDQ) | |
| X86.HasSSSE3 = isSet(ecx1, cpuid_SSSE3) | |
| X86.HasFMA = isSet(ecx1, cpuid_FMA) | |
| X86.HasCX16 = isSet(ecx1, cpuid_CX16) | |
| X86.HasSSE41 = isSet(ecx1, cpuid_SSE41) | |
| X86.HasSSE42 = isSet(ecx1, cpuid_SSE42) | |
| X86.HasPOPCNT = isSet(ecx1, cpuid_POPCNT) | |
| X86.HasAES = isSet(ecx1, cpuid_AES) | |
| X86.HasXSAVE = isSet(ecx1, cpuid_XSAVE) | |
| X86.HasOSXSAVE = isSet(ecx1, cpuid_OSXSAVE) | |
| X86.HasF16C = isSet(ecx1, cpuid_F16C) | |
| X86.HasMOVBE = isSet(ecx1, cpuid_MOVBE) | |
| X86.HasRDRAND = isSet(ecx1, cpuid_RDRAND) | |
| // Extended processor info and feature bits, under the separate extended | |
| // (0x8000_0000-prefixed) leaf hierarchy, so maxID (the basic leaf count) | |
| // doesn't gate it. | |
| if maxExtID, _, _, _ := cpuid(0x80000000, 0); maxExtID >= 0x80000001 { | |
| _, _, ecxExt1, _ := cpuid(0x80000001, 0) | |
| X86.HasLAHF = isSet(ecxExt1, cpuid_LAHF) | |
| X86.HasLZCNT = isSet(ecxExt1, cpuid_LZCNT) | |
| } | |
| var osSupportsAVX, osSupportsAVX512 bool | |
| // For XGETBV, OSXSAVE bit is required and sufficient. | |
| if X86.HasOSXSAVE { | |
| eax, _ := xgetbv() | |
| // Check if XMM and YMM registers have OS support. | |
| osSupportsAVX = isSet(eax, 1<<1) && isSet(eax, 1<<2) | |
| if runtime.GOOS == "darwin" { | |
| // Darwin requires special AVX512 checks, see cpu_darwin_x86.go | |
| osSupportsAVX512 = osSupportsAVX && darwinSupportsAVX512() | |
| } else { | |
| // Check if OPMASK and ZMM registers have OS support. | |
| osSupportsAVX512 = osSupportsAVX && isSet(eax, 1<<5) && isSet(eax, 1<<6) && isSet(eax, 1<<7) | |
| } | |
| } | |
| X86.HasAVX = isSet(ecx1, cpuid_AVX) && osSupportsAVX | |
| if maxID < 7 { | |
| return | |
| } | |
| eax7, ebx7, ecx7, edx7 := cpuid(7, 0) | |
| X86.HasBMI1 = isSet(ebx7, cpuid_BMI1) | |
| X86.HasAVX2 = isSet(ebx7, cpuid_AVX2) && osSupportsAVX | |
| X86.HasBMI2 = isSet(ebx7, cpuid_BMI2) | |
| X86.HasERMS = isSet(ebx7, cpuid_ERMS) | |
| X86.HasRDSEED = isSet(ebx7, cpuid_RDSEED) | |
| X86.HasADX = isSet(ebx7, cpuid_ADX) | |
| X86.HasAVX512 = isSet(ebx7, cpuid_AVX512F) && osSupportsAVX512 // Because avx-512 foundation is the core required extension | |
| if X86.HasAVX512 { | |
| X86.HasAVX512F = true | |
| X86.HasAVX512CD = isSet(ebx7, cpuid_AVX512CD) | |
| X86.HasAVX512ER = isSet(ebx7, cpuid_AVX512ER) | |
| X86.HasAVX512PF = isSet(ebx7, cpuid_AVX512PF) | |
| X86.HasAVX512VL = isSet(ebx7, cpuid_AVX512VL) | |
| X86.HasAVX512BW = isSet(ebx7, cpuid_AVX512BW) | |
| X86.HasAVX512DQ = isSet(ebx7, cpuid_AVX512DQ) | |
| X86.HasAVX512IFMA = isSet(ebx7, cpuid_AVX512IFMA) | |
| X86.HasAVX512VBMI = isSet(ecx7, cpuid_AVX512_VBMI) | |
| X86.HasAVX5124VNNIW = isSet(edx7, cpuid_AVX5124VNNIW) | |
| X86.HasAVX5124FMAPS = isSet(edx7, cpuid_AVX5124FMAPS) | |
| X86.HasAVX512VPOPCNTDQ = isSet(ecx7, cpuid_AVX512VPOPCNTDQ) | |
| X86.HasAVX512VPCLMULQDQ = isSet(ecx7, cpuid_AVX512VPCLMULQDQ) | |
| X86.HasAVX512VNNI = isSet(ecx7, cpuid_AVX512VNNI) | |
| X86.HasAVX512GFNI = isSet(ecx7, cpuid_AVX512GFNI) | |
| X86.HasAVX512VAES = isSet(ecx7, cpuid_AVX512VAES) | |
| X86.HasAVX512VBMI2 = isSet(ecx7, cpuid_AVX512VBMI2) | |
| X86.HasAVX512BITALG = isSet(ecx7, cpuid_AVX512BITALG) | |
| } | |
| X86.HasAMXTile = isSet(edx7, cpuid_AMXTile) | |
| X86.HasAMXInt8 = isSet(edx7, cpuid_AMXInt8) | |
| X86.HasAMXBF16 = isSet(edx7, cpuid_AMXBF16) | |
| // These features depend on the second level of extended features. | |
| if eax7 >= 1 { | |
| eax71, _, _, edx71 := cpuid(7, 1) | |
| if X86.HasAVX512 { | |
| X86.HasAVX512BF16 = isSet(eax71, cpuid_AVX512BF16) | |
| } | |
| if X86.HasAVX { | |
| X86.HasAVXIFMA = isSet(eax71, cpuid_AVXIFMA) | |
| X86.HasAVXVNNI = isSet(eax71, cpuid_AVXVNNI) | |
| X86.HasAVXVNNIInt8 = isSet(edx71, cpuid_AVXVNNIInt8) | |
| } | |
| } | |
| } | |
| func isSet(hwc uint32, value uint32) bool { | |
| return hwc&value != 0 | |
| } |