simd: add some inner-product benchmarks
Some hand-unrolling of important methods
was tried, and shown to improve results.
Results from
```
export GOEXPERIMENT=simd
for R in {1..25} ; do
go test -bench=B -run=none -count=1 -ldflags=-randlayout=$R . | tee -a arm64-128.log
GODEBUG=simd=0 go test -bench=B -run=none -count=1 -ldflags=-randlayout=$R . | tee -a arm64-0.log
GOARCH=amd64 go test -bench=B -run=none -count=1 -ldflags=-randlayout=$R . | tee -a amd64-128.log
GOARCH=amd64 GODEBUG=simd=+256 go test -bench=B -run=none -count=1 -ldflags=-randlayout=$R . | tee -a amd64-256.log
GOARCH=amd64 GODEBUG=simd=0 go test -bench=B -run=none -count=1 -ldflags=-randlayout=$R . | tee -a amd64-0.log
GOARCH=wasm GOOS=wasip1 go test -bench=B -run=none -count=1 -ldflags=-randlayout=$R . | tee -a wasm-128.log
GOARCH=wasm GOOS=wasip1 GODEBUG=simd=0 go test -bench=B -run=none -count=1 -ldflags=-randlayout=$R . | tee -a wasm-0.log
echo done with $R
done
```
run on a quiet-ish Apple laptop
then cleaned with
```
for i in *.log ; do sed -e '1,$s/-10//' -i "" $i; done
```
```
benchstat -ignore goarch,cpu,goos arm64*log amd64*log wasm*log
pkg: simd
│ arm64-0.log │ arm64-128.log │ amd64-0.log │ amd64-128.log │ amd64-256.log │ wasm-0.log │ wasm-128.log │
│ sec/op │ sec/op vs base │ sec/op vs base │ sec/op vs base │ sec/op vs base │ sec/op vs base │ sec/op vs base │
IPFMA 465.99µ ± 1% 68.17µ ± 0% -85.37% (p=0.000 n=25) 567.46µ ± 3% +21.78% (p=0.000 n=25) 126.51µ ± 0% -72.85% (p=0.000 n=25) 166.29µ ± 0% -64.31% (p=0.000 n=25) 2839.92µ ± 1% +509.44% (p=0.000 n=25) 156.27µ ± 0% -66.47% (p=0.000 n=25)
IP 533.06µ ± 2% 47.64µ ± 0% -91.06% (p=0.000 n=25) 645.00µ ± 0% +21.00% (p=0.000 n=25) 50.16µ ± 0% -90.59% (p=0.000 n=25) 49.39µ ± 0% -90.73% (p=0.000 n=25) 3702.28µ ± 1% +594.53% (p=0.000 n=25) 156.15µ ± 0% -70.71% (p=0.000 n=25)
IPUnroll 521.14µ ± 1% 27.92µ ± 0% -94.64% (p=0.000 n=25) 634.97µ ± 3% +21.84% (p=0.000 n=25) 37.38µ ± 1% -92.83% (p=0.000 n=25) 38.85µ ± 0% -92.54% (p=0.000 n=25) 4543.60µ ± 1% +771.86% (p=0.000 n=25) 97.75µ ± 0% -81.24% (p=0.000 n=25)
IPUnrollMore 531.67µ ± 2% 27.92µ ± 0% -94.75% (p=0.000 n=25) 634.78µ ± 2% +19.40% (p=0.000 n=25) 37.29µ ± 0% -92.99% (p=0.000 n=25) 42.76µ ± 0% -91.96% (p=0.000 n=25) 4679.65µ ± 1% +780.19% (p=0.000 n=25) 96.80µ ± 0% -81.79% (p=0.000 n=25)
IPnosimd0 722.6µ ± 1% 738.3µ ± 1% +2.18% (p=0.000 n=25) 930.1µ ± 1% +28.73% (p=0.000 n=25) 938.6µ ± 1% +29.89% (p=0.000 n=25) 931.8µ ± 1% +28.95% (p=0.000 n=25) 525.2µ ± 0% -27.32% (p=0.000 n=25) 525.2µ ± 0% -27.32% (p=0.000 n=25)
IPnosimd1 522.1µ ± 1% 526.0µ ± 1% +0.76% (p=0.011 n=25) 1049.7µ ± 1% +101.05% (p=0.000 n=25) 1049.5µ ± 1% +101.02% (p=0.000 n=25) 1049.8µ ± 1% +101.07% (p=0.000 n=25) 744.4µ ± 0% +42.58% (p=0.000 n=25) 744.7µ ± 0% +42.64% (p=0.000 n=25)
IPnosimdA 640.2µ ± 0% 643.0µ ± 1% ~ (p=0.088 n=25) 968.7µ ± 1% +51.31% (p=0.000 n=25) 964.3µ ± 1% +50.63% (p=0.000 n=25) 966.5µ ± 1% +50.97% (p=0.000 n=25) 541.7µ ± 0% -15.38% (p=0.000 n=25) 542.0µ ± 0% -15.34% (p=0.000 n=25)
IPnosimdAnotBloop 278.6µ ± 0% 280.1µ ± 0% +0.57% (p=0.022 n=25) 241.4µ ± 0% -13.34% (p=0.000 n=25) 241.4µ ± 0% -13.33% (p=0.000 n=25) 241.4µ ± 0% -13.35% (p=0.000 n=25) 392.2µ ± 0% +40.78% (p=0.000 n=25) 392.1µ ± 0% +40.78% (p=0.000 n=25)
geomean 510.7µ 143.2µ -71.96% 654.5µ +28.15% 194.3µ -61.96% 205.0µ -59.87% 1.441m +182.09% 257.3µ -49.63%
```
Change-Id: I09b912a3b68431972eeb91200f77a3dd95327d91
Reviewed-on: https://go-review.googlesource.com/c/go/+/787561
Reviewed-by: Junyang Shao <shaojunyang@google.com>
LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
diff --git a/src/simd/internal/bridge/simd_emulated.go b/src/simd/internal/bridge/simd_emulated.go
index 41b6417..4a82d16 100644
--- a/src/simd/internal/bridge/simd_emulated.go
+++ b/src/simd/internal/bridge/simd_emulated.go
@@ -2537,9 +2537,10 @@
// Add returns the element-wise sum of x and y.
func (x Float32s) Add(y Float32s) Float32s {
var res Float32s
- for i := 0; i < 4; i++ {
- res.set(i, x.get(i)+y.get(i))
- }
+ res.set(0, x.get(0)+y.get(0))
+ res.set(1, x.get(1)+y.get(1))
+ res.set(2, x.get(2)+y.get(2))
+ res.set(3, x.get(3)+y.get(3))
return res
}
@@ -2667,18 +2668,23 @@
// Mul returns the element-wise product of x and y.
func (x Float32s) Mul(y Float32s) Float32s {
var res Float32s
- for i := 0; i < 4; i++ {
- res.set(i, x.get(i)*y.get(i))
- }
+ res.set(0, x.get(0)*y.get(0))
+ res.set(1, x.get(1)*y.get(1))
+ res.set(2, x.get(2)*y.get(2))
+ res.set(3, x.get(3)*y.get(3))
+
return res
}
// MulAdd returns x * y + z element-wise.
func (x Float32s) MulAdd(y, z Float32s) Float32s {
var res Float32s
- for i := 0; i < 4; i++ {
- res.set(i, x.get(i)*y.get(i)+z.get(i))
- }
+
+ res.set(0, x.get(0)*y.get(0)+z.get(0))
+ res.set(1, x.get(1)*y.get(1)+z.get(1))
+ res.set(2, x.get(2)*y.get(2)+z.get(2))
+ res.set(3, x.get(3)*y.get(3)+z.get(3))
+
return res
}
diff --git a/src/simd/ip_test.go b/src/simd/ip_test.go
index 90aea58..b88c499 100644
--- a/src/simd/ip_test.go
+++ b/src/simd/ip_test.go
@@ -8,10 +8,62 @@
import (
"fmt"
+ "math/rand/v2"
"simd"
"testing"
)
+func fill(x, y []float32) {
+ for i := range x {
+ x[i] = 2*rand.Float32() - 1
+ y[i] = 2*rand.Float32() - 1
+ }
+}
+
+func checkErrors(b *testing.B, errors int) {
+ b.Helper()
+ if errors > 0 {
+ b.Logf("errors = %d", errors)
+ }
+}
+
+// BenchmarkIPFMA is simd vector inner product computing using FMA.
+func BenchmarkIPFMA(b *testing.B) {
+ x := make([]float32, ipBenchLen)
+ y := make([]float32, ipBenchLen)
+
+ fill(x, y)
+
+ ip0, _, _ := ipFMA(x, y)
+
+ var errors int
+ for b.Loop() {
+ z, _, _ := ipFMA(x, y)
+ if z != ip0 {
+ errors++
+ }
+ }
+ checkErrors(b, errors)
+}
+
+func ipFMA(x, y []float32) (float32, int, bool) {
+ var a simd.Float32s
+ sumWidth := a.Len() * 32
+ emulated := simd.Emulated()
+ var i int
+ for i = 0; i < len(x)-a.Len()+1; i += a.Len() {
+ u := simd.LoadFloat32s(x[i : i+a.Len()])
+ v := simd.LoadFloat32s(y[i : i+a.Len()])
+ a = u.MulAdd(v, a)
+ }
+ if i < len(x) {
+ a = first(simd.LoadFloat32sPart(x[i:])).MulAdd(
+ first(simd.LoadFloat32sPart(y[i:])), a)
+ }
+
+ return sum(a), sumWidth, emulated
+}
+
func TestIP(t *testing.T) {
var a, b [50]float32
@@ -48,6 +100,173 @@
return t
}
+const ipBenchLen = 300000
+
+// BenchmarkIP is simd vector inner product, vanilla transcription.
+func BenchmarkIP(b *testing.B) {
+ x := make([]float32, ipBenchLen)
+ y := make([]float32, ipBenchLen)
+
+ fill(x, y)
+
+ ip0, _, _ := ip(x, y)
+
+ var errors int
+ for b.Loop() {
+ z, _, _ := ip(x, y)
+ if z != ip0 {
+ errors++
+ }
+ }
+ checkErrors(b, errors)
+}
+
+// BenchmarkIPUnroll is simd vector inner product, unrolled 4x vector ops.
+func BenchmarkIPUnroll(b *testing.B) {
+ x := make([]float32, ipBenchLen)
+ y := make([]float32, ipBenchLen)
+
+ fill(x, y)
+
+ ip0, _, _ := ipU(x, y)
+
+ var errors int
+ for b.Loop() {
+ z, _, _ := ipU(x, y)
+ if z != ip0 {
+ errors++
+ }
+ }
+ checkErrors(b, errors)
+}
+
+// BenchmarkIPUnrollMore is simd vector inner product, unrolled 5x vector ops
+func BenchmarkIPUnrollMore(b *testing.B) {
+ x := make([]float32, ipBenchLen)
+ y := make([]float32, ipBenchLen)
+
+ fill(x, y)
+
+ ip0, _, _ := ipUmore(x, y)
+
+ var errors int
+ for b.Loop() {
+ z, _, _ := ipUmore(x, y)
+ if z != ip0 {
+ errors++
+ }
+ }
+ checkErrors(b, errors)
+}
+
+// ipNosimd computes inner product with serial
+// addition order of the terms (to make the)
+// check comparison turn out right.
+func ipNosimd(x, y []float32) float32 {
+ var z float32
+ for i, a := range x {
+ z += a * y[i]
+ }
+ return z
+}
+
+// BenchmarkIPnosimd1 is serial, just a vanilla inner product.
+func BenchmarkIPnosimd0(b *testing.B) {
+ x := make([]float32, ipBenchLen)
+ y := make([]float32, ipBenchLen)
+
+ fill(x, y)
+
+ ip0 := ipNosimd(x, y)
+
+ var errors int
+ for b.Loop() {
+ var z float32
+ for i, a := range x {
+ z += a * y[i]
+ }
+ if z != ip0 {
+ errors++
+ }
+ }
+ checkErrors(b, errors)
+}
+
+// BenchmarkIPnosimd1 is serial, but with a no-op subslice that
+// makes it clear that x and y have the same length.
+func BenchmarkIPnosimd1(b *testing.B) {
+ x := make([]float32, ipBenchLen)
+ y := make([]float32, ipBenchLen)
+
+ fill(x, y)
+
+ ip0 := ipNosimd(x, y)
+
+ var errors int
+ for b.Loop() {
+ var z float32
+ yy := y[:(len(x))]
+ for i, a := range x {
+ z += a * yy[i]
+ }
+ if z != ip0 {
+ errors++
+ }
+ }
+ checkErrors(b, errors)
+}
+
+// BenchmarkIPnosimdA is serial, rewritten to use arrays instead of slices,
+// so no bounds checking, gosh darn it to heck.
+func BenchmarkIPnosimdA(b *testing.B) {
+ var x, y [ipBenchLen]float32
+
+ fill(x[:], y[:])
+
+ ip0 := ipNosimd(x[:], y[:])
+
+ var errors int
+ for b.Loop() {
+ var z float32
+ for i, a := range x {
+ z += a * y[i]
+ }
+ if z != ip0 {
+ errors++
+ }
+ }
+ checkErrors(b, errors)
+}
+
+var x, y [ipBenchLen]float32
+var ip0 float32
+
+func initIp0() {
+ fill(x[:], y[:])
+ ip0 = ipNosimd(x[:], y[:])
+}
+
+// BenchmarkIPnosimdAnotBloop is serial, rewritten to use arrays instead of slices,
+// and using a classic iterated loop to see if b.Loop affects subscript inference,
+// so no bounds checking, gosh darn it to heck, this time, for sure.
+func BenchmarkIPnosimdAnotBloop(b *testing.B) {
+ if ip0 == 0 {
+ initIp0()
+ }
+
+ var errors int
+ for range b.N {
+ var z float32
+ for i, a := range x {
+ z += a * y[i]
+ }
+ if z != ip0 {
+ errors++
+ }
+ }
+ checkErrors(b, errors)
+}
+
func ip(x, y []float32) (float32, int, bool) {
var a simd.Float32s
sumWidth := a.Len() * 32
@@ -66,6 +285,96 @@
return sum(a), sumWidth, emulated
}
+func ipU(x, y []float32) (float32, int, bool) {
+ const U = 4
+ var a, a0, a1, a2, a3 simd.Float32s
+ sumWidth := a.Len() * 32
+ emulated := simd.Emulated()
+ var i int
+ for i = 0; i < len(x)-U*a.Len()+1; i += U * a.Len() {
+ i0 := i
+ i1 := i + a.Len()
+ i2 := i + 2*a.Len()
+ i3 := i + 3*a.Len()
+
+ u := simd.LoadFloat32s(x[i0 : i0+a.Len()])
+ v := simd.LoadFloat32s(y[i0 : i0+a.Len()])
+ a0 = a0.Add(u.Mul(v))
+
+ u = simd.LoadFloat32s(x[i1 : i1+a.Len()])
+ v = simd.LoadFloat32s(y[i1 : i1+a.Len()])
+ a1 = a1.Add(u.Mul(v))
+
+ u = simd.LoadFloat32s(x[i2 : i2+a.Len()])
+ v = simd.LoadFloat32s(y[i2 : i2+a.Len()])
+ a2 = a2.Add(u.Mul(v))
+
+ u = simd.LoadFloat32s(x[i3 : i3+a.Len()])
+ v = simd.LoadFloat32s(y[i3 : i3+a.Len()])
+ a3 = a3.Add(u.Mul(v))
+ }
+ a = a0.Add(a1).Add(a2.Add(a3))
+ for ; i < len(x)-a.Len()+1; i += a.Len() {
+ u := simd.LoadFloat32s(x[i : i+a.Len()])
+ v := simd.LoadFloat32s(y[i : i+a.Len()])
+ a = a.Add(u.Mul(v))
+ }
+ if i < len(x) {
+ a = a.Add(first(simd.LoadFloat32sPart(x[i:])).
+ Mul(first(simd.LoadFloat32sPart(y[i:]))))
+ }
+
+ return sum(a), sumWidth, emulated
+}
+
+func ipUmore(x, y []float32) (float32, int, bool) {
+ const U = 5
+ var a, a0, a1, a2, a3, a4 simd.Float32s
+ sumWidth := a.Len() * 32
+ emulated := simd.Emulated()
+ var i int
+ for i = 0; i < len(x)-U*a.Len()+1; i += U * a.Len() {
+ i0 := i
+ i1 := i + a.Len()
+ i2 := i + 2*a.Len()
+ i3 := i + 3*a.Len()
+ i4 := i + 4*a.Len()
+
+ u := simd.LoadFloat32s(x[i0 : i0+a.Len()])
+ v := simd.LoadFloat32s(y[i0 : i0+a.Len()])
+ a0 = a0.Add(u.Mul(v))
+
+ u = simd.LoadFloat32s(x[i1 : i1+a.Len()])
+ v = simd.LoadFloat32s(y[i1 : i1+a.Len()])
+ a1 = a1.Add(u.Mul(v))
+
+ u = simd.LoadFloat32s(x[i2 : i2+a.Len()])
+ v = simd.LoadFloat32s(y[i2 : i2+a.Len()])
+ a2 = a2.Add(u.Mul(v))
+
+ u = simd.LoadFloat32s(x[i3 : i3+a.Len()])
+ v = simd.LoadFloat32s(y[i3 : i3+a.Len()])
+ a3 = a3.Add(u.Mul(v))
+
+ u = simd.LoadFloat32s(x[i4 : i4+a.Len()])
+ v = simd.LoadFloat32s(y[i4 : i4+a.Len()])
+ a4 = a4.Add(u.Mul(v))
+ }
+ a = a0.Add(a1).Add(a2.Add(a3)).Add(a4)
+
+ for ; i < len(x)-a.Len()+1; i += a.Len() {
+ u := simd.LoadFloat32s(x[i : i+a.Len()])
+ v := simd.LoadFloat32s(y[i : i+a.Len()])
+ a = a.Add(u.Mul(v))
+ }
+ if i < len(x) {
+ a = a.Add(first(simd.LoadFloat32sPart(x[i:])).
+ Mul(first(simd.LoadFloat32sPart(y[i:]))))
+ }
+
+ return sum(a), sumWidth, emulated
+}
+
func ipGoTo(x, y []float32) (float32, int, bool) {
var a simd.Float32s
sumWidth := a.Len() * 32
diff --git a/src/simd/simd_emulated.go b/src/simd/simd_emulated.go
index 1ad562e..48f09d5 100644
--- a/src/simd/simd_emulated.go
+++ b/src/simd/simd_emulated.go
@@ -2537,9 +2537,10 @@
// Add returns the element-wise sum of x and y.
func (x Float32s) Add(y Float32s) Float32s {
var res Float32s
- for i := 0; i < 4; i++ {
- res.set(i, x.get(i)+y.get(i))
- }
+ res.set(0, x.get(0)+y.get(0))
+ res.set(1, x.get(1)+y.get(1))
+ res.set(2, x.get(2)+y.get(2))
+ res.set(3, x.get(3)+y.get(3))
return res
}
@@ -2667,18 +2668,22 @@
// Mul returns the element-wise product of x and y.
func (x Float32s) Mul(y Float32s) Float32s {
var res Float32s
- for i := 0; i < 4; i++ {
- res.set(i, x.get(i)*y.get(i))
- }
+ res.set(0, x.get(0)*y.get(0))
+ res.set(1, x.get(1)*y.get(1))
+ res.set(2, x.get(2)*y.get(2))
+ res.set(3, x.get(3)*y.get(3))
+
return res
}
// MulAdd returns x * y + z element-wise.
func (x Float32s) MulAdd(y, z Float32s) Float32s {
var res Float32s
- for i := 0; i < 4; i++ {
- res.set(i, x.get(i)*y.get(i)+z.get(i))
- }
+
+ res.set(0, x.get(0)*y.get(0)+z.get(0))
+ res.set(1, x.get(1)*y.get(1)+z.get(1))
+ res.set(2, x.get(2)*y.get(2)+z.get(2))
+ res.set(3, x.get(3)*y.get(3)+z.get(3))
return res
}
