simd, cmd/compile: add 8-bit interleave operations on amd64 Add InterleaveLo and InterleaveHi for Int8x16 and Uint8x16, and add InterleaveLoGrouped and InterleaveHiGrouped for Int8x32, Int8x64, Uint8x32, and Uint8x64 on amd64. Lower them to VPUNPCKLBW and VPUNPCKHBW, respectively, and add coverage for signed and unsigned vectors. Change-Id: I5bf45e9b318890181ef376876251e31e1512bc98 Reviewed-on: https://go-review.googlesource.com/c/go/+/796340 Reviewed-by: David Chase <drchase@google.com> Reviewed-by: Cherry Mui <cherryyz@google.com> LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
diff --git a/src/cmd/compile/internal/amd64/simdssa.go b/src/cmd/compile/internal/amd64/simdssa.go index 7700be1..7c6872f 100644 --- a/src/cmd/compile/internal/amd64/simdssa.go +++ b/src/cmd/compile/internal/amd64/simdssa.go
@@ -331,18 +331,24 @@ ssaop.OpAMD64VPCMPGTD256, ssaop.OpAMD64VPCMPGTQ128, ssaop.OpAMD64VPCMPGTQ256, + ssaop.OpAMD64VPUNPCKHBW128, ssaop.OpAMD64VPUNPCKHWD128, ssaop.OpAMD64VPUNPCKHDQ128, ssaop.OpAMD64VPUNPCKHQDQ128, + ssaop.OpAMD64VPUNPCKHBW256, + ssaop.OpAMD64VPUNPCKHBW512, ssaop.OpAMD64VPUNPCKHWD256, ssaop.OpAMD64VPUNPCKHWD512, ssaop.OpAMD64VPUNPCKHDQ256, ssaop.OpAMD64VPUNPCKHDQ512, ssaop.OpAMD64VPUNPCKHQDQ256, ssaop.OpAMD64VPUNPCKHQDQ512, + ssaop.OpAMD64VPUNPCKLBW128, ssaop.OpAMD64VPUNPCKLWD128, ssaop.OpAMD64VPUNPCKLDQ128, ssaop.OpAMD64VPUNPCKLQDQ128, + ssaop.OpAMD64VPUNPCKLBW256, + ssaop.OpAMD64VPUNPCKLBW512, ssaop.OpAMD64VPUNPCKLWD256, ssaop.OpAMD64VPUNPCKLWD512, ssaop.OpAMD64VPUNPCKLDQ256, @@ -2006,17 +2012,21 @@ ssaop.OpAMD64VPCMPGTD256load, ssaop.OpAMD64VPCMPGTQ128load, ssaop.OpAMD64VPCMPGTQ256load, + ssaop.OpAMD64VPUNPCKHBW128load, ssaop.OpAMD64VPUNPCKHWD128load, ssaop.OpAMD64VPUNPCKHDQ128load, ssaop.OpAMD64VPUNPCKHQDQ128load, + ssaop.OpAMD64VPUNPCKHBW256load, ssaop.OpAMD64VPUNPCKHWD256load, ssaop.OpAMD64VPUNPCKHDQ256load, ssaop.OpAMD64VPUNPCKHDQ512load, ssaop.OpAMD64VPUNPCKHQDQ256load, ssaop.OpAMD64VPUNPCKHQDQ512load, + ssaop.OpAMD64VPUNPCKLBW128load, ssaop.OpAMD64VPUNPCKLWD128load, ssaop.OpAMD64VPUNPCKLDQ128load, ssaop.OpAMD64VPUNPCKLQDQ128load, + ssaop.OpAMD64VPUNPCKLBW256load, ssaop.OpAMD64VPUNPCKLWD256load, ssaop.OpAMD64VPUNPCKLDQ256load, ssaop.OpAMD64VPUNPCKLDQ512load,
diff --git a/src/cmd/compile/internal/ssa/_gen/simdAMD64.rules b/src/cmd/compile/internal/ssa/_gen/simdAMD64.rules index 389f4c3..6943010 100644 --- a/src/cmd/compile/internal/ssa/_gen/simdAMD64.rules +++ b/src/cmd/compile/internal/ssa/_gen/simdAMD64.rules
@@ -511,36 +511,48 @@ (GreaterEqualUint16x32 x y) => (VPMOVMToVec16x32 (VPCMPUW512 [13] x y)) // maskOut (GreaterEqualUint32x16 x y) => (VPMOVMToVec32x16 (VPCMPUD512 [13] x y)) // maskOut (GreaterEqualUint64x8 x y) => (VPMOVMToVec64x8 (VPCMPUQ512 [13] x y)) // maskOut +(InterleaveHiInt8x16 ...) => (VPUNPCKHBW128 ...) // pureVreg (InterleaveHiInt16x8 ...) => (VPUNPCKHWD128 ...) // pureVreg (InterleaveHiInt32x4 ...) => (VPUNPCKHDQ128 ...) // pureVreg (InterleaveHiInt64x2 ...) => (VPUNPCKHQDQ128 ...) // pureVreg +(InterleaveHiUint8x16 ...) => (VPUNPCKHBW128 ...) // pureVreg (InterleaveHiUint16x8 ...) => (VPUNPCKHWD128 ...) // pureVreg (InterleaveHiUint32x4 ...) => (VPUNPCKHDQ128 ...) // pureVreg (InterleaveHiUint64x2 ...) => (VPUNPCKHQDQ128 ...) // pureVreg +(InterleaveHiGroupedInt8x32 ...) => (VPUNPCKHBW256 ...) // pureVreg +(InterleaveHiGroupedInt8x64 ...) => (VPUNPCKHBW512 ...) // pureVreg (InterleaveHiGroupedInt16x16 ...) => (VPUNPCKHWD256 ...) // pureVreg (InterleaveHiGroupedInt16x32 ...) => (VPUNPCKHWD512 ...) // pureVreg (InterleaveHiGroupedInt32x8 ...) => (VPUNPCKHDQ256 ...) // pureVreg (InterleaveHiGroupedInt32x16 ...) => (VPUNPCKHDQ512 ...) // pureVreg (InterleaveHiGroupedInt64x4 ...) => (VPUNPCKHQDQ256 ...) // pureVreg (InterleaveHiGroupedInt64x8 ...) => (VPUNPCKHQDQ512 ...) // pureVreg +(InterleaveHiGroupedUint8x32 ...) => (VPUNPCKHBW256 ...) // pureVreg +(InterleaveHiGroupedUint8x64 ...) => (VPUNPCKHBW512 ...) // pureVreg (InterleaveHiGroupedUint16x16 ...) => (VPUNPCKHWD256 ...) // pureVreg (InterleaveHiGroupedUint16x32 ...) => (VPUNPCKHWD512 ...) // pureVreg (InterleaveHiGroupedUint32x8 ...) => (VPUNPCKHDQ256 ...) // pureVreg (InterleaveHiGroupedUint32x16 ...) => (VPUNPCKHDQ512 ...) // pureVreg (InterleaveHiGroupedUint64x4 ...) => (VPUNPCKHQDQ256 ...) // pureVreg (InterleaveHiGroupedUint64x8 ...) => (VPUNPCKHQDQ512 ...) // pureVreg +(InterleaveLoInt8x16 ...) => (VPUNPCKLBW128 ...) // pureVreg (InterleaveLoInt16x8 ...) => (VPUNPCKLWD128 ...) // pureVreg (InterleaveLoInt32x4 ...) => (VPUNPCKLDQ128 ...) // pureVreg (InterleaveLoInt64x2 ...) => (VPUNPCKLQDQ128 ...) // pureVreg +(InterleaveLoUint8x16 ...) => (VPUNPCKLBW128 ...) // pureVreg (InterleaveLoUint16x8 ...) => (VPUNPCKLWD128 ...) // pureVreg (InterleaveLoUint32x4 ...) => (VPUNPCKLDQ128 ...) // pureVreg (InterleaveLoUint64x2 ...) => (VPUNPCKLQDQ128 ...) // pureVreg +(InterleaveLoGroupedInt8x32 ...) => (VPUNPCKLBW256 ...) // pureVreg +(InterleaveLoGroupedInt8x64 ...) => (VPUNPCKLBW512 ...) // pureVreg (InterleaveLoGroupedInt16x16 ...) => (VPUNPCKLWD256 ...) // pureVreg (InterleaveLoGroupedInt16x32 ...) => (VPUNPCKLWD512 ...) // pureVreg (InterleaveLoGroupedInt32x8 ...) => (VPUNPCKLDQ256 ...) // pureVreg (InterleaveLoGroupedInt32x16 ...) => (VPUNPCKLDQ512 ...) // pureVreg (InterleaveLoGroupedInt64x4 ...) => (VPUNPCKLQDQ256 ...) // pureVreg (InterleaveLoGroupedInt64x8 ...) => (VPUNPCKLQDQ512 ...) // pureVreg +(InterleaveLoGroupedUint8x32 ...) => (VPUNPCKLBW256 ...) // pureVreg +(InterleaveLoGroupedUint8x64 ...) => (VPUNPCKLBW512 ...) // pureVreg (InterleaveLoGroupedUint16x16 ...) => (VPUNPCKLWD256 ...) // pureVreg (InterleaveLoGroupedUint16x32 ...) => (VPUNPCKLWD512 ...) // pureVreg (InterleaveLoGroupedUint32x8 ...) => (VPUNPCKLDQ256 ...) // pureVreg @@ -2971,17 +2983,21 @@ (VPCMPUQ512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPCMPUQ512load {sym} [ssa.MakeValAndOff(int32(uint8(c)),off)] x ptr mem) // vregMem (VPCMPD512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPCMPD512load {sym} [ssa.MakeValAndOff(int32(uint8(c)),off)] x ptr mem) // vregMem (VPCMPQ512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPCMPQ512load {sym} [ssa.MakeValAndOff(int32(uint8(c)),off)] x ptr mem) // vregMem +(VPUNPCKHBW128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKHBW128load {sym} [off] x ptr mem) // vregMem (VPUNPCKHWD128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKHWD128load {sym} [off] x ptr mem) // vregMem (VPUNPCKHDQ128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKHDQ128load {sym} [off] x ptr mem) // vregMem (VPUNPCKHQDQ128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKHQDQ128load {sym} [off] x ptr mem) // vregMem +(VPUNPCKHBW256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKHBW256load {sym} [off] x ptr mem) // vregMem (VPUNPCKHWD256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKHWD256load {sym} [off] x ptr mem) // vregMem (VPUNPCKHDQ256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKHDQ256load {sym} [off] x ptr mem) // vregMem (VPUNPCKHDQ512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKHDQ512load {sym} [off] x ptr mem) // vregMem (VPUNPCKHQDQ256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKHQDQ256load {sym} [off] x ptr mem) // vregMem (VPUNPCKHQDQ512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKHQDQ512load {sym} [off] x ptr mem) // vregMem +(VPUNPCKLBW128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKLBW128load {sym} [off] x ptr mem) // vregMem (VPUNPCKLWD128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKLWD128load {sym} [off] x ptr mem) // vregMem (VPUNPCKLDQ128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKLDQ128load {sym} [off] x ptr mem) // vregMem (VPUNPCKLQDQ128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKLQDQ128load {sym} [off] x ptr mem) // vregMem +(VPUNPCKLBW256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKLBW256load {sym} [off] x ptr mem) // vregMem (VPUNPCKLWD256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKLWD256load {sym} [off] x ptr mem) // vregMem (VPUNPCKLDQ256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKLDQ256load {sym} [off] x ptr mem) // vregMem (VPUNPCKLDQ512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPUNPCKLDQ512load {sym} [off] x ptr mem) // vregMem
diff --git a/src/cmd/compile/internal/ssa/_gen/simdAMD64ops.go b/src/cmd/compile/internal/ssa/_gen/simdAMD64ops.go index ccb988d..d9337cc 100644 --- a/src/cmd/compile/internal/ssa/_gen/simdAMD64ops.go +++ b/src/cmd/compile/internal/ssa/_gen/simdAMD64ops.go
@@ -1145,6 +1145,9 @@ {name: "VPSUBWMasked128", argLength: 3, reg: w2kw, asm: "VPSUBW", typ: "Vec128"}, {name: "VPSUBWMasked256", argLength: 3, reg: w2kw, asm: "VPSUBW", typ: "Vec256"}, {name: "VPSUBWMasked512", argLength: 3, reg: w2kw, asm: "VPSUBW", typ: "Vec512"}, + {name: "VPUNPCKHBW128", argLength: 2, reg: v21, asm: "VPUNPCKHBW", typ: "Vec128"}, + {name: "VPUNPCKHBW256", argLength: 2, reg: v21, asm: "VPUNPCKHBW", typ: "Vec256"}, + {name: "VPUNPCKHBW512", argLength: 2, reg: w21, asm: "VPUNPCKHBW", typ: "Vec512"}, {name: "VPUNPCKHDQ128", argLength: 2, reg: v21, asm: "VPUNPCKHDQ", typ: "Vec128"}, {name: "VPUNPCKHDQ256", argLength: 2, reg: v21, asm: "VPUNPCKHDQ", typ: "Vec256"}, {name: "VPUNPCKHDQ512", argLength: 2, reg: w21, asm: "VPUNPCKHDQ", typ: "Vec512"}, @@ -1154,6 +1157,9 @@ {name: "VPUNPCKHWD128", argLength: 2, reg: v21, asm: "VPUNPCKHWD", typ: "Vec128"}, {name: "VPUNPCKHWD256", argLength: 2, reg: v21, asm: "VPUNPCKHWD", typ: "Vec256"}, {name: "VPUNPCKHWD512", argLength: 2, reg: w21, asm: "VPUNPCKHWD", typ: "Vec512"}, + {name: "VPUNPCKLBW128", argLength: 2, reg: v21, asm: "VPUNPCKLBW", typ: "Vec128"}, + {name: "VPUNPCKLBW256", argLength: 2, reg: v21, asm: "VPUNPCKLBW", typ: "Vec256"}, + {name: "VPUNPCKLBW512", argLength: 2, reg: w21, asm: "VPUNPCKLBW", typ: "Vec512"}, {name: "VPUNPCKLDQ128", argLength: 2, reg: v21, asm: "VPUNPCKLDQ", typ: "Vec128"}, {name: "VPUNPCKLDQ256", argLength: 2, reg: v21, asm: "VPUNPCKLDQ", typ: "Vec256"}, {name: "VPUNPCKLDQ512", argLength: 2, reg: w21, asm: "VPUNPCKLDQ", typ: "Vec512"}, @@ -2323,6 +2329,8 @@ {name: "VPSUBW256load", argLength: 3, reg: v21load, asm: "VPSUBW", typ: "Vec256", aux: "SymOff", symEffect: "Read"}, {name: "VPSUBWMasked128load", argLength: 4, reg: w2kwload, asm: "VPSUBW", typ: "Vec128", aux: "SymOff", symEffect: "Read"}, {name: "VPSUBWMasked256load", argLength: 4, reg: w2kwload, asm: "VPSUBW", typ: "Vec256", aux: "SymOff", symEffect: "Read"}, + {name: "VPUNPCKHBW128load", argLength: 3, reg: v21load, asm: "VPUNPCKHBW", typ: "Vec128", aux: "SymOff", symEffect: "Read"}, + {name: "VPUNPCKHBW256load", argLength: 3, reg: v21load, asm: "VPUNPCKHBW", typ: "Vec256", aux: "SymOff", symEffect: "Read"}, {name: "VPUNPCKHDQ128load", argLength: 3, reg: v21load, asm: "VPUNPCKHDQ", typ: "Vec128", aux: "SymOff", symEffect: "Read"}, {name: "VPUNPCKHDQ256load", argLength: 3, reg: v21load, asm: "VPUNPCKHDQ", typ: "Vec256", aux: "SymOff", symEffect: "Read"}, {name: "VPUNPCKHDQ512load", argLength: 3, reg: w21load, asm: "VPUNPCKHDQ", typ: "Vec512", aux: "SymOff", symEffect: "Read"}, @@ -2331,6 +2339,8 @@ {name: "VPUNPCKHQDQ512load", argLength: 3, reg: w21load, asm: "VPUNPCKHQDQ", typ: "Vec512", aux: "SymOff", symEffect: "Read"}, {name: "VPUNPCKHWD128load", argLength: 3, reg: v21load, asm: "VPUNPCKHWD", typ: "Vec128", aux: "SymOff", symEffect: "Read"}, {name: "VPUNPCKHWD256load", argLength: 3, reg: v21load, asm: "VPUNPCKHWD", typ: "Vec256", aux: "SymOff", symEffect: "Read"}, + {name: "VPUNPCKLBW128load", argLength: 3, reg: v21load, asm: "VPUNPCKLBW", typ: "Vec128", aux: "SymOff", symEffect: "Read"}, + {name: "VPUNPCKLBW256load", argLength: 3, reg: v21load, asm: "VPUNPCKLBW", typ: "Vec256", aux: "SymOff", symEffect: "Read"}, {name: "VPUNPCKLDQ128load", argLength: 3, reg: v21load, asm: "VPUNPCKLDQ", typ: "Vec128", aux: "SymOff", symEffect: "Read"}, {name: "VPUNPCKLDQ256load", argLength: 3, reg: v21load, asm: "VPUNPCKLDQ", typ: "Vec256", aux: "SymOff", symEffect: "Read"}, {name: "VPUNPCKLDQ512load", argLength: 3, reg: w21load, asm: "VPUNPCKLDQ", typ: "Vec512", aux: "SymOff", symEffect: "Read"},
diff --git a/src/cmd/compile/internal/ssa/_gen/simdgenericOps.go b/src/cmd/compile/internal/ssa/_gen/simdgenericOps.go index 155f8b0..a82ea95 100644 --- a/src/cmd/compile/internal/ssa/_gen/simdgenericOps.go +++ b/src/cmd/compile/internal/ssa/_gen/simdgenericOps.go
@@ -554,43 +554,51 @@ {name: "InterleaveEvenUint16x8", argLength: 2}, // ARCH:arm64 {name: "InterleaveEvenUint32x4", argLength: 2}, // ARCH:arm64 {name: "InterleaveEvenUint64x2", argLength: 2}, // ARCH:arm64 + {name: "InterleaveHiGroupedInt8x32", argLength: 2}, // ARCH:amd64 + {name: "InterleaveHiGroupedInt8x64", argLength: 2}, // ARCH:amd64 {name: "InterleaveHiGroupedInt16x16", argLength: 2}, // ARCH:amd64 {name: "InterleaveHiGroupedInt16x32", argLength: 2}, // ARCH:amd64 {name: "InterleaveHiGroupedInt32x8", argLength: 2}, // ARCH:amd64 {name: "InterleaveHiGroupedInt32x16", argLength: 2}, // ARCH:amd64 {name: "InterleaveHiGroupedInt64x4", argLength: 2}, // ARCH:amd64 {name: "InterleaveHiGroupedInt64x8", argLength: 2}, // ARCH:amd64 + {name: "InterleaveHiGroupedUint8x32", argLength: 2}, // ARCH:amd64 + {name: "InterleaveHiGroupedUint8x64", argLength: 2}, // ARCH:amd64 {name: "InterleaveHiGroupedUint16x16", argLength: 2}, // ARCH:amd64 {name: "InterleaveHiGroupedUint16x32", argLength: 2}, // ARCH:amd64 {name: "InterleaveHiGroupedUint32x8", argLength: 2}, // ARCH:amd64 {name: "InterleaveHiGroupedUint32x16", argLength: 2}, // ARCH:amd64 {name: "InterleaveHiGroupedUint64x4", argLength: 2}, // ARCH:amd64 {name: "InterleaveHiGroupedUint64x8", argLength: 2}, // ARCH:amd64 - {name: "InterleaveHiInt8x16", argLength: 2}, // ARCH:arm64 + {name: "InterleaveHiInt8x16", argLength: 2}, // ARCH:amd64,arm64 {name: "InterleaveHiInt16x8", argLength: 2}, // ARCH:amd64,arm64 {name: "InterleaveHiInt32x4", argLength: 2}, // ARCH:amd64,arm64 {name: "InterleaveHiInt64x2", argLength: 2}, // ARCH:amd64,arm64 - {name: "InterleaveHiUint8x16", argLength: 2}, // ARCH:arm64 + {name: "InterleaveHiUint8x16", argLength: 2}, // ARCH:amd64,arm64 {name: "InterleaveHiUint16x8", argLength: 2}, // ARCH:amd64,arm64 {name: "InterleaveHiUint32x4", argLength: 2}, // ARCH:amd64,arm64 {name: "InterleaveHiUint64x2", argLength: 2}, // ARCH:amd64,arm64 + {name: "InterleaveLoGroupedInt8x32", argLength: 2}, // ARCH:amd64 + {name: "InterleaveLoGroupedInt8x64", argLength: 2}, // ARCH:amd64 {name: "InterleaveLoGroupedInt16x16", argLength: 2}, // ARCH:amd64 {name: "InterleaveLoGroupedInt16x32", argLength: 2}, // ARCH:amd64 {name: "InterleaveLoGroupedInt32x8", argLength: 2}, // ARCH:amd64 {name: "InterleaveLoGroupedInt32x16", argLength: 2}, // ARCH:amd64 {name: "InterleaveLoGroupedInt64x4", argLength: 2}, // ARCH:amd64 {name: "InterleaveLoGroupedInt64x8", argLength: 2}, // ARCH:amd64 + {name: "InterleaveLoGroupedUint8x32", argLength: 2}, // ARCH:amd64 + {name: "InterleaveLoGroupedUint8x64", argLength: 2}, // ARCH:amd64 {name: "InterleaveLoGroupedUint16x16", argLength: 2}, // ARCH:amd64 {name: "InterleaveLoGroupedUint16x32", argLength: 2}, // ARCH:amd64 {name: "InterleaveLoGroupedUint32x8", argLength: 2}, // ARCH:amd64 {name: "InterleaveLoGroupedUint32x16", argLength: 2}, // ARCH:amd64 {name: "InterleaveLoGroupedUint64x4", argLength: 2}, // ARCH:amd64 {name: "InterleaveLoGroupedUint64x8", argLength: 2}, // ARCH:amd64 - {name: "InterleaveLoInt8x16", argLength: 2}, // ARCH:arm64 + {name: "InterleaveLoInt8x16", argLength: 2}, // ARCH:amd64,arm64 {name: "InterleaveLoInt16x8", argLength: 2}, // ARCH:amd64,arm64 {name: "InterleaveLoInt32x4", argLength: 2}, // ARCH:amd64,arm64 {name: "InterleaveLoInt64x2", argLength: 2}, // ARCH:amd64,arm64 - {name: "InterleaveLoUint8x16", argLength: 2}, // ARCH:arm64 + {name: "InterleaveLoUint8x16", argLength: 2}, // ARCH:amd64,arm64 {name: "InterleaveLoUint16x8", argLength: 2}, // ARCH:amd64,arm64 {name: "InterleaveLoUint32x4", argLength: 2}, // ARCH:amd64,arm64 {name: "InterleaveLoUint64x2", argLength: 2}, // ARCH:amd64,arm64
diff --git a/src/cmd/compile/internal/ssa/ssaop/opGen.go b/src/cmd/compile/internal/ssa/ssaop/opGen.go index 231a6ac..2aa43b9 100644 --- a/src/cmd/compile/internal/ssa/ssaop/opGen.go +++ b/src/cmd/compile/internal/ssa/ssaop/opGen.go
@@ -2111,6 +2111,9 @@ OpAMD64VPSUBWMasked128 OpAMD64VPSUBWMasked256 OpAMD64VPSUBWMasked512 + OpAMD64VPUNPCKHBW128 + OpAMD64VPUNPCKHBW256 + OpAMD64VPUNPCKHBW512 OpAMD64VPUNPCKHDQ128 OpAMD64VPUNPCKHDQ256 OpAMD64VPUNPCKHDQ512 @@ -2120,6 +2123,9 @@ OpAMD64VPUNPCKHWD128 OpAMD64VPUNPCKHWD256 OpAMD64VPUNPCKHWD512 + OpAMD64VPUNPCKLBW128 + OpAMD64VPUNPCKLBW256 + OpAMD64VPUNPCKLBW512 OpAMD64VPUNPCKLDQ128 OpAMD64VPUNPCKLDQ256 OpAMD64VPUNPCKLDQ512 @@ -3289,6 +3295,8 @@ OpAMD64VPSUBW256load OpAMD64VPSUBWMasked128load OpAMD64VPSUBWMasked256load + OpAMD64VPUNPCKHBW128load + OpAMD64VPUNPCKHBW256load OpAMD64VPUNPCKHDQ128load OpAMD64VPUNPCKHDQ256load OpAMD64VPUNPCKHDQ512load @@ -3297,6 +3305,8 @@ OpAMD64VPUNPCKHQDQ512load OpAMD64VPUNPCKHWD128load OpAMD64VPUNPCKHWD256load + OpAMD64VPUNPCKLBW128load + OpAMD64VPUNPCKLBW256load OpAMD64VPUNPCKLDQ128load OpAMD64VPUNPCKLDQ256load OpAMD64VPUNPCKLDQ512load @@ -7501,12 +7511,16 @@ OpInterleaveEvenUint16x8 OpInterleaveEvenUint32x4 OpInterleaveEvenUint64x2 + OpInterleaveHiGroupedInt8x32 + OpInterleaveHiGroupedInt8x64 OpInterleaveHiGroupedInt16x16 OpInterleaveHiGroupedInt16x32 OpInterleaveHiGroupedInt32x8 OpInterleaveHiGroupedInt32x16 OpInterleaveHiGroupedInt64x4 OpInterleaveHiGroupedInt64x8 + OpInterleaveHiGroupedUint8x32 + OpInterleaveHiGroupedUint8x64 OpInterleaveHiGroupedUint16x16 OpInterleaveHiGroupedUint16x32 OpInterleaveHiGroupedUint32x8 @@ -7521,12 +7535,16 @@ OpInterleaveHiUint16x8 OpInterleaveHiUint32x4 OpInterleaveHiUint64x2 + OpInterleaveLoGroupedInt8x32 + OpInterleaveLoGroupedInt8x64 OpInterleaveLoGroupedInt16x16 OpInterleaveLoGroupedInt16x32 OpInterleaveLoGroupedInt32x8 OpInterleaveLoGroupedInt32x16 OpInterleaveLoGroupedInt64x4 OpInterleaveLoGroupedInt64x8 + OpInterleaveLoGroupedUint8x32 + OpInterleaveLoGroupedUint8x64 OpInterleaveLoGroupedUint16x16 OpInterleaveLoGroupedUint16x32 OpInterleaveLoGroupedUint32x8 @@ -40105,6 +40123,48 @@ }, }, { + Name: "VPUNPCKHBW128", + ArgLen: 2, + asm: x86.AVPUNPCKHBW, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 2147418112, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, RegMask{V1: 4294901760, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 2147418112, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + }, + }, + }, + { + Name: "VPUNPCKHBW256", + ArgLen: 2, + asm: x86.AVPUNPCKHBW, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 2147418112, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, RegMask{V1: 4294901760, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 2147418112, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + }, + }, + }, + { + Name: "VPUNPCKHBW512", + ArgLen: 2, + asm: x86.AVPUNPCKHBW, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 281474976645120, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {1, RegMask{V1: 281474976645120, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 281472829161472, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + }, + }, + }, + { Name: "VPUNPCKHDQ128", ArgLen: 2, asm: x86.AVPUNPCKHDQ, @@ -40231,6 +40291,48 @@ }, }, { + Name: "VPUNPCKLBW128", + ArgLen: 2, + asm: x86.AVPUNPCKLBW, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 2147418112, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, RegMask{V1: 4294901760, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 2147418112, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + }, + }, + }, + { + Name: "VPUNPCKLBW256", + ArgLen: 2, + asm: x86.AVPUNPCKLBW, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 2147418112, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, RegMask{V1: 4294901760, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 2147418112, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + }, + }, + }, + { + Name: "VPUNPCKLBW512", + ArgLen: 2, + asm: x86.AVPUNPCKLBW, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 281474976645120, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {1, RegMask{V1: 281474976645120, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 281472829161472, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + }, + }, + }, + { Name: "VPUNPCKLDQ128", ArgLen: 2, asm: x86.AVPUNPCKLDQ, @@ -58868,6 +58970,38 @@ }, }, { + Name: "VPUNPCKHBW128load", + AuxType: AuxTypeSymOff, + ArgLen: 3, + symEffect: SymRead, + asm: x86.AVPUNPCKHBW, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 2147418112, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, RegMask{V1: 72057594037977087, V2: 0}}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R15 SB + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 2147418112, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + }, + }, + }, + { + Name: "VPUNPCKHBW256load", + AuxType: AuxTypeSymOff, + ArgLen: 3, + symEffect: SymRead, + asm: x86.AVPUNPCKHBW, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 2147418112, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, RegMask{V1: 72057594037977087, V2: 0}}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R15 SB + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 2147418112, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + }, + }, + }, + { Name: "VPUNPCKHDQ128load", AuxType: AuxTypeSymOff, ArgLen: 3, @@ -58996,6 +59130,38 @@ }, }, { + Name: "VPUNPCKLBW128load", + AuxType: AuxTypeSymOff, + ArgLen: 3, + symEffect: SymRead, + asm: x86.AVPUNPCKLBW, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 2147418112, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, RegMask{V1: 72057594037977087, V2: 0}}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R15 SB + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 2147418112, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + }, + }, + }, + { + Name: "VPUNPCKLBW256load", + AuxType: AuxTypeSymOff, + ArgLen: 3, + symEffect: SymRead, + asm: x86.AVPUNPCKLBW, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 2147418112, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, RegMask{V1: 72057594037977087, V2: 0}}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R15 SB + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 2147418112, V2: 0}}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + }, + }, + }, + { Name: "VPUNPCKLDQ128load", AuxType: AuxTypeSymOff, ArgLen: 3, @@ -111981,6 +112147,16 @@ Generic: true, }, { + Name: "InterleaveHiGroupedInt8x32", + ArgLen: 2, + Generic: true, + }, + { + Name: "InterleaveHiGroupedInt8x64", + ArgLen: 2, + Generic: true, + }, + { Name: "InterleaveHiGroupedInt16x16", ArgLen: 2, Generic: true, @@ -112011,6 +112187,16 @@ Generic: true, }, { + Name: "InterleaveHiGroupedUint8x32", + ArgLen: 2, + Generic: true, + }, + { + Name: "InterleaveHiGroupedUint8x64", + ArgLen: 2, + Generic: true, + }, + { Name: "InterleaveHiGroupedUint16x16", ArgLen: 2, Generic: true, @@ -112081,6 +112267,16 @@ Generic: true, }, { + Name: "InterleaveLoGroupedInt8x32", + ArgLen: 2, + Generic: true, + }, + { + Name: "InterleaveLoGroupedInt8x64", + ArgLen: 2, + Generic: true, + }, + { Name: "InterleaveLoGroupedInt16x16", ArgLen: 2, Generic: true, @@ -112111,6 +112307,16 @@ Generic: true, }, { + Name: "InterleaveLoGroupedUint8x32", + ArgLen: 2, + Generic: true, + }, + { + Name: "InterleaveLoGroupedUint8x64", + ArgLen: 2, + Generic: true, + }, + { Name: "InterleaveLoGroupedUint16x16", ArgLen: 2, Generic: true,
diff --git a/src/cmd/compile/internal/ssagen/simdAMD64intrinsics.go b/src/cmd/compile/internal/ssagen/simdAMD64intrinsics.go index 889c96c..545c5a3 100644 --- a/src/cmd/compile/internal/ssagen/simdAMD64intrinsics.go +++ b/src/cmd/compile/internal/ssagen/simdAMD64intrinsics.go
@@ -522,36 +522,48 @@ addF(simdPackage, "Uint16x32.GreaterEqual", opLen2(ssaop.OpGreaterEqualUint16x32, types.TypeVec512), sys.AMD64) addF(simdPackage, "Uint32x16.GreaterEqual", opLen2(ssaop.OpGreaterEqualUint32x16, types.TypeVec512), sys.AMD64) addF(simdPackage, "Uint64x8.GreaterEqual", opLen2(ssaop.OpGreaterEqualUint64x8, types.TypeVec512), sys.AMD64) + addF(simdPackage, "Int8x16.InterleaveHi", opLen2(ssaop.OpInterleaveHiInt8x16, types.TypeVec128), sys.AMD64) addF(simdPackage, "Int16x8.InterleaveHi", opLen2(ssaop.OpInterleaveHiInt16x8, types.TypeVec128), sys.AMD64) addF(simdPackage, "Int32x4.InterleaveHi", opLen2(ssaop.OpInterleaveHiInt32x4, types.TypeVec128), sys.AMD64) addF(simdPackage, "Int64x2.InterleaveHi", opLen2(ssaop.OpInterleaveHiInt64x2, types.TypeVec128), sys.AMD64) + addF(simdPackage, "Uint8x16.InterleaveHi", opLen2(ssaop.OpInterleaveHiUint8x16, types.TypeVec128), sys.AMD64) addF(simdPackage, "Uint16x8.InterleaveHi", opLen2(ssaop.OpInterleaveHiUint16x8, types.TypeVec128), sys.AMD64) addF(simdPackage, "Uint32x4.InterleaveHi", opLen2(ssaop.OpInterleaveHiUint32x4, types.TypeVec128), sys.AMD64) addF(simdPackage, "Uint64x2.InterleaveHi", opLen2(ssaop.OpInterleaveHiUint64x2, types.TypeVec128), sys.AMD64) + addF(simdPackage, "Int8x32.InterleaveHiGrouped", opLen2(ssaop.OpInterleaveHiGroupedInt8x32, types.TypeVec256), sys.AMD64) + addF(simdPackage, "Int8x64.InterleaveHiGrouped", opLen2(ssaop.OpInterleaveHiGroupedInt8x64, types.TypeVec512), sys.AMD64) addF(simdPackage, "Int16x16.InterleaveHiGrouped", opLen2(ssaop.OpInterleaveHiGroupedInt16x16, types.TypeVec256), sys.AMD64) addF(simdPackage, "Int16x32.InterleaveHiGrouped", opLen2(ssaop.OpInterleaveHiGroupedInt16x32, types.TypeVec512), sys.AMD64) addF(simdPackage, "Int32x8.InterleaveHiGrouped", opLen2(ssaop.OpInterleaveHiGroupedInt32x8, types.TypeVec256), sys.AMD64) addF(simdPackage, "Int32x16.InterleaveHiGrouped", opLen2(ssaop.OpInterleaveHiGroupedInt32x16, types.TypeVec512), sys.AMD64) addF(simdPackage, "Int64x4.InterleaveHiGrouped", opLen2(ssaop.OpInterleaveHiGroupedInt64x4, types.TypeVec256), sys.AMD64) addF(simdPackage, "Int64x8.InterleaveHiGrouped", opLen2(ssaop.OpInterleaveHiGroupedInt64x8, types.TypeVec512), sys.AMD64) + addF(simdPackage, "Uint8x32.InterleaveHiGrouped", opLen2(ssaop.OpInterleaveHiGroupedUint8x32, types.TypeVec256), sys.AMD64) + addF(simdPackage, "Uint8x64.InterleaveHiGrouped", opLen2(ssaop.OpInterleaveHiGroupedUint8x64, types.TypeVec512), sys.AMD64) addF(simdPackage, "Uint16x16.InterleaveHiGrouped", opLen2(ssaop.OpInterleaveHiGroupedUint16x16, types.TypeVec256), sys.AMD64) addF(simdPackage, "Uint16x32.InterleaveHiGrouped", opLen2(ssaop.OpInterleaveHiGroupedUint16x32, types.TypeVec512), sys.AMD64) addF(simdPackage, "Uint32x8.InterleaveHiGrouped", opLen2(ssaop.OpInterleaveHiGroupedUint32x8, types.TypeVec256), sys.AMD64) addF(simdPackage, "Uint32x16.InterleaveHiGrouped", opLen2(ssaop.OpInterleaveHiGroupedUint32x16, types.TypeVec512), sys.AMD64) addF(simdPackage, "Uint64x4.InterleaveHiGrouped", opLen2(ssaop.OpInterleaveHiGroupedUint64x4, types.TypeVec256), sys.AMD64) addF(simdPackage, "Uint64x8.InterleaveHiGrouped", opLen2(ssaop.OpInterleaveHiGroupedUint64x8, types.TypeVec512), sys.AMD64) + addF(simdPackage, "Int8x16.InterleaveLo", opLen2(ssaop.OpInterleaveLoInt8x16, types.TypeVec128), sys.AMD64) addF(simdPackage, "Int16x8.InterleaveLo", opLen2(ssaop.OpInterleaveLoInt16x8, types.TypeVec128), sys.AMD64) addF(simdPackage, "Int32x4.InterleaveLo", opLen2(ssaop.OpInterleaveLoInt32x4, types.TypeVec128), sys.AMD64) addF(simdPackage, "Int64x2.InterleaveLo", opLen2(ssaop.OpInterleaveLoInt64x2, types.TypeVec128), sys.AMD64) + addF(simdPackage, "Uint8x16.InterleaveLo", opLen2(ssaop.OpInterleaveLoUint8x16, types.TypeVec128), sys.AMD64) addF(simdPackage, "Uint16x8.InterleaveLo", opLen2(ssaop.OpInterleaveLoUint16x8, types.TypeVec128), sys.AMD64) addF(simdPackage, "Uint32x4.InterleaveLo", opLen2(ssaop.OpInterleaveLoUint32x4, types.TypeVec128), sys.AMD64) addF(simdPackage, "Uint64x2.InterleaveLo", opLen2(ssaop.OpInterleaveLoUint64x2, types.TypeVec128), sys.AMD64) + addF(simdPackage, "Int8x32.InterleaveLoGrouped", opLen2(ssaop.OpInterleaveLoGroupedInt8x32, types.TypeVec256), sys.AMD64) + addF(simdPackage, "Int8x64.InterleaveLoGrouped", opLen2(ssaop.OpInterleaveLoGroupedInt8x64, types.TypeVec512), sys.AMD64) addF(simdPackage, "Int16x16.InterleaveLoGrouped", opLen2(ssaop.OpInterleaveLoGroupedInt16x16, types.TypeVec256), sys.AMD64) addF(simdPackage, "Int16x32.InterleaveLoGrouped", opLen2(ssaop.OpInterleaveLoGroupedInt16x32, types.TypeVec512), sys.AMD64) addF(simdPackage, "Int32x8.InterleaveLoGrouped", opLen2(ssaop.OpInterleaveLoGroupedInt32x8, types.TypeVec256), sys.AMD64) addF(simdPackage, "Int32x16.InterleaveLoGrouped", opLen2(ssaop.OpInterleaveLoGroupedInt32x16, types.TypeVec512), sys.AMD64) addF(simdPackage, "Int64x4.InterleaveLoGrouped", opLen2(ssaop.OpInterleaveLoGroupedInt64x4, types.TypeVec256), sys.AMD64) addF(simdPackage, "Int64x8.InterleaveLoGrouped", opLen2(ssaop.OpInterleaveLoGroupedInt64x8, types.TypeVec512), sys.AMD64) + addF(simdPackage, "Uint8x32.InterleaveLoGrouped", opLen2(ssaop.OpInterleaveLoGroupedUint8x32, types.TypeVec256), sys.AMD64) + addF(simdPackage, "Uint8x64.InterleaveLoGrouped", opLen2(ssaop.OpInterleaveLoGroupedUint8x64, types.TypeVec512), sys.AMD64) addF(simdPackage, "Uint16x16.InterleaveLoGrouped", opLen2(ssaop.OpInterleaveLoGroupedUint16x16, types.TypeVec256), sys.AMD64) addF(simdPackage, "Uint16x32.InterleaveLoGrouped", opLen2(ssaop.OpInterleaveLoGroupedUint16x32, types.TypeVec512), sys.AMD64) addF(simdPackage, "Uint32x8.InterleaveLoGrouped", opLen2(ssaop.OpInterleaveLoGroupedUint32x8, types.TypeVec256), sys.AMD64)
diff --git a/src/cmd/compile/internal/ssarewrite/rewriteamd64/rewriteAMD64.go b/src/cmd/compile/internal/ssarewrite/rewriteamd64/rewriteAMD64.go index 1394f56..14254c4 100644 --- a/src/cmd/compile/internal/ssarewrite/rewriteamd64/rewriteAMD64.go +++ b/src/cmd/compile/internal/ssarewrite/rewriteamd64/rewriteAMD64.go
@@ -2827,6 +2827,10 @@ return rewriteValue_OpAMD64VPTERNLOGQ256(v) case ssaop.OpAMD64VPTERNLOGQ512: return rewriteValue_OpAMD64VPTERNLOGQ512(v) + case ssaop.OpAMD64VPUNPCKHBW128: + return rewriteValue_OpAMD64VPUNPCKHBW128(v) + case ssaop.OpAMD64VPUNPCKHBW256: + return rewriteValue_OpAMD64VPUNPCKHBW256(v) case ssaop.OpAMD64VPUNPCKHDQ128: return rewriteValue_OpAMD64VPUNPCKHDQ128(v) case ssaop.OpAMD64VPUNPCKHDQ256: @@ -2843,6 +2847,10 @@ return rewriteValue_OpAMD64VPUNPCKHWD128(v) case ssaop.OpAMD64VPUNPCKHWD256: return rewriteValue_OpAMD64VPUNPCKHWD256(v) + case ssaop.OpAMD64VPUNPCKLBW128: + return rewriteValue_OpAMD64VPUNPCKLBW128(v) + case ssaop.OpAMD64VPUNPCKLBW256: + return rewriteValue_OpAMD64VPUNPCKLBW256(v) case ssaop.OpAMD64VPUNPCKLDQ128: return rewriteValue_OpAMD64VPUNPCKLDQ128(v) case ssaop.OpAMD64VPUNPCKLDQ256: @@ -4748,6 +4756,12 @@ case ssaop.OpInterleaveHiGroupedInt64x8: v.Op = ssaop.OpAMD64VPUNPCKHQDQ512 return true + case ssaop.OpInterleaveHiGroupedInt8x32: + v.Op = ssaop.OpAMD64VPUNPCKHBW256 + return true + case ssaop.OpInterleaveHiGroupedInt8x64: + v.Op = ssaop.OpAMD64VPUNPCKHBW512 + return true case ssaop.OpInterleaveHiGroupedUint16x16: v.Op = ssaop.OpAMD64VPUNPCKHWD256 return true @@ -4766,6 +4780,12 @@ case ssaop.OpInterleaveHiGroupedUint64x8: v.Op = ssaop.OpAMD64VPUNPCKHQDQ512 return true + case ssaop.OpInterleaveHiGroupedUint8x32: + v.Op = ssaop.OpAMD64VPUNPCKHBW256 + return true + case ssaop.OpInterleaveHiGroupedUint8x64: + v.Op = ssaop.OpAMD64VPUNPCKHBW512 + return true case ssaop.OpInterleaveHiInt16x8: v.Op = ssaop.OpAMD64VPUNPCKHWD128 return true @@ -4775,6 +4795,9 @@ case ssaop.OpInterleaveHiInt64x2: v.Op = ssaop.OpAMD64VPUNPCKHQDQ128 return true + case ssaop.OpInterleaveHiInt8x16: + v.Op = ssaop.OpAMD64VPUNPCKHBW128 + return true case ssaop.OpInterleaveHiUint16x8: v.Op = ssaop.OpAMD64VPUNPCKHWD128 return true @@ -4784,6 +4807,9 @@ case ssaop.OpInterleaveHiUint64x2: v.Op = ssaop.OpAMD64VPUNPCKHQDQ128 return true + case ssaop.OpInterleaveHiUint8x16: + v.Op = ssaop.OpAMD64VPUNPCKHBW128 + return true case ssaop.OpInterleaveLoGroupedInt16x16: v.Op = ssaop.OpAMD64VPUNPCKLWD256 return true @@ -4802,6 +4828,12 @@ case ssaop.OpInterleaveLoGroupedInt64x8: v.Op = ssaop.OpAMD64VPUNPCKLQDQ512 return true + case ssaop.OpInterleaveLoGroupedInt8x32: + v.Op = ssaop.OpAMD64VPUNPCKLBW256 + return true + case ssaop.OpInterleaveLoGroupedInt8x64: + v.Op = ssaop.OpAMD64VPUNPCKLBW512 + return true case ssaop.OpInterleaveLoGroupedUint16x16: v.Op = ssaop.OpAMD64VPUNPCKLWD256 return true @@ -4820,6 +4852,12 @@ case ssaop.OpInterleaveLoGroupedUint64x8: v.Op = ssaop.OpAMD64VPUNPCKLQDQ512 return true + case ssaop.OpInterleaveLoGroupedUint8x32: + v.Op = ssaop.OpAMD64VPUNPCKLBW256 + return true + case ssaop.OpInterleaveLoGroupedUint8x64: + v.Op = ssaop.OpAMD64VPUNPCKLBW512 + return true case ssaop.OpInterleaveLoInt16x8: v.Op = ssaop.OpAMD64VPUNPCKLWD128 return true @@ -4829,6 +4867,9 @@ case ssaop.OpInterleaveLoInt64x2: v.Op = ssaop.OpAMD64VPUNPCKLQDQ128 return true + case ssaop.OpInterleaveLoInt8x16: + v.Op = ssaop.OpAMD64VPUNPCKLBW128 + return true case ssaop.OpInterleaveLoUint16x8: v.Op = ssaop.OpAMD64VPUNPCKLWD128 return true @@ -4838,6 +4879,9 @@ case ssaop.OpInterleaveLoUint64x2: v.Op = ssaop.OpAMD64VPUNPCKLQDQ128 return true + case ssaop.OpInterleaveLoUint8x16: + v.Op = ssaop.OpAMD64VPUNPCKLBW128 + return true case ssaop.OpIsInBounds: return rewriteValue_OpIsInBounds(v) case ssaop.OpIsNaNFloat32x16: @@ -90280,6 +90324,60 @@ } return false } +func rewriteValue_OpAMD64VPUNPCKHBW128(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (VPUNPCKHBW128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) + // cond: ssa.CanMergeLoad(v, l) && ssa.Clobber(l) + // result: (VPUNPCKHBW128load {sym} [off] x ptr mem) + for { + x := v_0 + l := v_1 + if l.Op != ssaop.OpAMD64VMOVDQUload128 { + break + } + off := ssa.AuxIntToInt32(l.AuxInt) + sym := ssa.AuxToSym(l.Aux) + mem := l.Args[1] + ptr := l.Args[0] + if !(ssa.CanMergeLoad(v, l) && ssa.Clobber(l)) { + break + } + v.Reset(ssaop.OpAMD64VPUNPCKHBW128load) + v.AuxInt = ssa.Int32ToAuxInt(off) + v.Aux = ssa.SymToAux(sym) + v.AddArg3(x, ptr, mem) + return true + } + return false +} +func rewriteValue_OpAMD64VPUNPCKHBW256(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (VPUNPCKHBW256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) + // cond: ssa.CanMergeLoad(v, l) && ssa.Clobber(l) + // result: (VPUNPCKHBW256load {sym} [off] x ptr mem) + for { + x := v_0 + l := v_1 + if l.Op != ssaop.OpAMD64VMOVDQUload256 { + break + } + off := ssa.AuxIntToInt32(l.AuxInt) + sym := ssa.AuxToSym(l.Aux) + mem := l.Args[1] + ptr := l.Args[0] + if !(ssa.CanMergeLoad(v, l) && ssa.Clobber(l)) { + break + } + v.Reset(ssaop.OpAMD64VPUNPCKHBW256load) + v.AuxInt = ssa.Int32ToAuxInt(off) + v.Aux = ssa.SymToAux(sym) + v.AddArg3(x, ptr, mem) + return true + } + return false +} func rewriteValue_OpAMD64VPUNPCKHDQ128(v *ssa.Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] @@ -90496,6 +90594,60 @@ } return false } +func rewriteValue_OpAMD64VPUNPCKLBW128(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (VPUNPCKLBW128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) + // cond: ssa.CanMergeLoad(v, l) && ssa.Clobber(l) + // result: (VPUNPCKLBW128load {sym} [off] x ptr mem) + for { + x := v_0 + l := v_1 + if l.Op != ssaop.OpAMD64VMOVDQUload128 { + break + } + off := ssa.AuxIntToInt32(l.AuxInt) + sym := ssa.AuxToSym(l.Aux) + mem := l.Args[1] + ptr := l.Args[0] + if !(ssa.CanMergeLoad(v, l) && ssa.Clobber(l)) { + break + } + v.Reset(ssaop.OpAMD64VPUNPCKLBW128load) + v.AuxInt = ssa.Int32ToAuxInt(off) + v.Aux = ssa.SymToAux(sym) + v.AddArg3(x, ptr, mem) + return true + } + return false +} +func rewriteValue_OpAMD64VPUNPCKLBW256(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (VPUNPCKLBW256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) + // cond: ssa.CanMergeLoad(v, l) && ssa.Clobber(l) + // result: (VPUNPCKLBW256load {sym} [off] x ptr mem) + for { + x := v_0 + l := v_1 + if l.Op != ssaop.OpAMD64VMOVDQUload256 { + break + } + off := ssa.AuxIntToInt32(l.AuxInt) + sym := ssa.AuxToSym(l.Aux) + mem := l.Args[1] + ptr := l.Args[0] + if !(ssa.CanMergeLoad(v, l) && ssa.Clobber(l)) { + break + } + v.Reset(ssaop.OpAMD64VPUNPCKLBW256load) + v.AuxInt = ssa.Int32ToAuxInt(off) + v.Aux = ssa.SymToAux(sym) + v.AddArg3(x, ptr, mem) + return true + } + return false +} func rewriteValue_OpAMD64VPUNPCKLDQ128(v *ssa.Value) bool { v_1 := v.Args[1] v_0 := v.Args[0]
diff --git a/src/simd/archsimd/_gen/simdgen/ops/Moves/go_amd64.yaml b/src/simd/archsimd/_gen/simdgen/ops/Moves/go_amd64.yaml index aeb99b6..3a7d47b 100644 --- a/src/simd/archsimd/_gen/simdgen/ops/Moves/go_amd64.yaml +++ b/src/simd/archsimd/_gen/simdgen/ops/Moves/go_amd64.yaml
@@ -744,7 +744,7 @@ - *256Or512lanes8 - go: InterleaveHi - asm: VPUNPCKH(QDQ|DQ|WD|WB) + asm: VPUNPCKH(QDQ|DQ|WD|BW) in: - *128any - *128any @@ -753,7 +753,7 @@ - *128any - go: InterleaveLo - asm: VPUNPCKL(QDQ|DQ|WD|WB) + asm: VPUNPCKL(QDQ|DQ|WD|BW) in: - *128any - *128any @@ -762,7 +762,7 @@ - *128any - go: InterleaveHiGrouped - asm: VPUNPCKH(QDQ|DQ|WD|WB) + asm: VPUNPCKH(QDQ|DQ|WD|BW) in: - *256Or512any - *256Or512any @@ -771,7 +771,7 @@ - *256Or512any - go: InterleaveLoGrouped - asm: VPUNPCKL(QDQ|DQ|WD|WB) + asm: VPUNPCKL(QDQ|DQ|WD|BW) in: - *256Or512any - *256Or512any
diff --git a/src/simd/archsimd/internal/simd_test/binary_amd64_test.go b/src/simd/archsimd/internal/simd_test/binary_amd64_test.go index 412f7a3..28607fc 100644 --- a/src/simd/archsimd/internal/simd_test/binary_amd64_test.go +++ b/src/simd/archsimd/internal/simd_test/binary_amd64_test.go
@@ -290,26 +290,32 @@ } func TestInterleaveLo(t *testing.T) { + testInt8x16Binary(t, archsimd.Int8x16.InterleaveLo, interleaveSlice[int8](128, false)) testInt16x8Binary(t, archsimd.Int16x8.InterleaveLo, interleaveSlice[int16](128, false)) testInt32x4Binary(t, archsimd.Int32x4.InterleaveLo, interleaveSlice[int32](128, false)) testInt64x2Binary(t, archsimd.Int64x2.InterleaveLo, interleaveSlice[int64](128, false)) + testUint8x16Binary(t, archsimd.Uint8x16.InterleaveLo, interleaveSlice[uint8](128, false)) testUint16x8Binary(t, archsimd.Uint16x8.InterleaveLo, interleaveSlice[uint16](128, false)) testUint32x4Binary(t, archsimd.Uint32x4.InterleaveLo, interleaveSlice[uint32](128, false)) testUint64x2Binary(t, archsimd.Uint64x2.InterleaveLo, interleaveSlice[uint64](128, false)) if archsimd.X86.AVX2() { + testInt8x32Binary(t, archsimd.Int8x32.InterleaveLoGrouped, interleaveSlice[int8](128, false)) testInt16x16Binary(t, archsimd.Int16x16.InterleaveLoGrouped, interleaveSlice[int16](128, false)) testInt32x8Binary(t, archsimd.Int32x8.InterleaveLoGrouped, interleaveSlice[int32](128, false)) testInt64x4Binary(t, archsimd.Int64x4.InterleaveLoGrouped, interleaveSlice[int64](128, false)) + testUint8x32Binary(t, archsimd.Uint8x32.InterleaveLoGrouped, interleaveSlice[uint8](128, false)) testUint16x16Binary(t, archsimd.Uint16x16.InterleaveLoGrouped, interleaveSlice[uint16](128, false)) testUint32x8Binary(t, archsimd.Uint32x8.InterleaveLoGrouped, interleaveSlice[uint32](128, false)) testUint64x4Binary(t, archsimd.Uint64x4.InterleaveLoGrouped, interleaveSlice[uint64](128, false)) } if archsimd.X86.AVX512() { + testInt8x64Binary(t, archsimd.Int8x64.InterleaveLoGrouped, interleaveSlice[int8](128, false)) testInt16x32Binary(t, archsimd.Int16x32.InterleaveLoGrouped, interleaveSlice[int16](128, false)) testInt32x16Binary(t, archsimd.Int32x16.InterleaveLoGrouped, interleaveSlice[int32](128, false)) testInt64x8Binary(t, archsimd.Int64x8.InterleaveLoGrouped, interleaveSlice[int64](128, false)) + testUint8x64Binary(t, archsimd.Uint8x64.InterleaveLoGrouped, interleaveSlice[uint8](128, false)) testUint16x32Binary(t, archsimd.Uint16x32.InterleaveLoGrouped, interleaveSlice[uint16](128, false)) testUint32x16Binary(t, archsimd.Uint32x16.InterleaveLoGrouped, interleaveSlice[uint32](128, false)) testUint64x8Binary(t, archsimd.Uint64x8.InterleaveLoGrouped, interleaveSlice[uint64](128, false)) @@ -317,26 +323,32 @@ } func TestInterleaveHi(t *testing.T) { + testInt8x16Binary(t, archsimd.Int8x16.InterleaveHi, interleaveSlice[int8](128, true)) testInt16x8Binary(t, archsimd.Int16x8.InterleaveHi, interleaveSlice[int16](128, true)) testInt32x4Binary(t, archsimd.Int32x4.InterleaveHi, interleaveSlice[int32](128, true)) testInt64x2Binary(t, archsimd.Int64x2.InterleaveHi, interleaveSlice[int64](128, true)) + testUint8x16Binary(t, archsimd.Uint8x16.InterleaveHi, interleaveSlice[uint8](128, true)) testUint16x8Binary(t, archsimd.Uint16x8.InterleaveHi, interleaveSlice[uint16](128, true)) testUint32x4Binary(t, archsimd.Uint32x4.InterleaveHi, interleaveSlice[uint32](128, true)) testUint64x2Binary(t, archsimd.Uint64x2.InterleaveHi, interleaveSlice[uint64](128, true)) if archsimd.X86.AVX2() { + testInt8x32Binary(t, archsimd.Int8x32.InterleaveHiGrouped, interleaveSlice[int8](128, true)) testInt16x16Binary(t, archsimd.Int16x16.InterleaveHiGrouped, interleaveSlice[int16](128, true)) testInt32x8Binary(t, archsimd.Int32x8.InterleaveHiGrouped, interleaveSlice[int32](128, true)) testInt64x4Binary(t, archsimd.Int64x4.InterleaveHiGrouped, interleaveSlice[int64](128, true)) + testUint8x32Binary(t, archsimd.Uint8x32.InterleaveHiGrouped, interleaveSlice[uint8](128, true)) testUint16x16Binary(t, archsimd.Uint16x16.InterleaveHiGrouped, interleaveSlice[uint16](128, true)) testUint32x8Binary(t, archsimd.Uint32x8.InterleaveHiGrouped, interleaveSlice[uint32](128, true)) testUint64x4Binary(t, archsimd.Uint64x4.InterleaveHiGrouped, interleaveSlice[uint64](128, true)) } if archsimd.X86.AVX512() { + testInt8x64Binary(t, archsimd.Int8x64.InterleaveHiGrouped, interleaveSlice[int8](128, true)) testInt16x32Binary(t, archsimd.Int16x32.InterleaveHiGrouped, interleaveSlice[int16](128, true)) testInt32x16Binary(t, archsimd.Int32x16.InterleaveHiGrouped, interleaveSlice[int32](128, true)) testInt64x8Binary(t, archsimd.Int64x8.InterleaveHiGrouped, interleaveSlice[int64](128, true)) + testUint8x64Binary(t, archsimd.Uint8x64.InterleaveHiGrouped, interleaveSlice[uint8](128, true)) testUint16x32Binary(t, archsimd.Uint16x32.InterleaveHiGrouped, interleaveSlice[uint16](128, true)) testUint32x16Binary(t, archsimd.Uint32x16.InterleaveHiGrouped, interleaveSlice[uint32](128, true)) testUint64x8Binary(t, archsimd.Uint64x8.InterleaveHiGrouped, interleaveSlice[uint64](128, true))
diff --git a/src/simd/archsimd/ops_amd64.go b/src/simd/archsimd/ops_amd64.go index 6782676..f93313d 100644 --- a/src/simd/archsimd/ops_amd64.go +++ b/src/simd/archsimd/ops_amd64.go
@@ -3279,6 +3279,11 @@ // InterleaveHi interleaves the elements of the high halves of x and y. // +// Asm: VPUNPCKHBW, CPU Feature: AVX +func (x Int8x16) InterleaveHi(y Int8x16) Int8x16 + +// InterleaveHi interleaves the elements of the high halves of x and y. +// // Asm: VPUNPCKHWD, CPU Feature: AVX func (x Int16x8) InterleaveHi(y Int16x8) Int16x8 @@ -3294,6 +3299,11 @@ // InterleaveHi interleaves the elements of the high halves of x and y. // +// Asm: VPUNPCKHBW, CPU Feature: AVX +func (x Uint8x16) InterleaveHi(y Uint8x16) Uint8x16 + +// InterleaveHi interleaves the elements of the high halves of x and y. +// // Asm: VPUNPCKHWD, CPU Feature: AVX func (x Uint16x8) InterleaveHi(y Uint16x8) Uint16x8 @@ -3311,6 +3321,16 @@ // InterleaveHiGrouped interleaves the elements of the high half of each 128-bit subvector of x and y. // +// Asm: VPUNPCKHBW, CPU Feature: AVX2 +func (x Int8x32) InterleaveHiGrouped(y Int8x32) Int8x32 + +// InterleaveHiGrouped interleaves the elements of the high half of each 128-bit subvector of x and y. +// +// Asm: VPUNPCKHBW, CPU Feature: AVX512 +func (x Int8x64) InterleaveHiGrouped(y Int8x64) Int8x64 + +// InterleaveHiGrouped interleaves the elements of the high half of each 128-bit subvector of x and y. +// // Asm: VPUNPCKHWD, CPU Feature: AVX2 func (x Int16x16) InterleaveHiGrouped(y Int16x16) Int16x16 @@ -3341,6 +3361,16 @@ // InterleaveHiGrouped interleaves the elements of the high half of each 128-bit subvector of x and y. // +// Asm: VPUNPCKHBW, CPU Feature: AVX2 +func (x Uint8x32) InterleaveHiGrouped(y Uint8x32) Uint8x32 + +// InterleaveHiGrouped interleaves the elements of the high half of each 128-bit subvector of x and y. +// +// Asm: VPUNPCKHBW, CPU Feature: AVX512 +func (x Uint8x64) InterleaveHiGrouped(y Uint8x64) Uint8x64 + +// InterleaveHiGrouped interleaves the elements of the high half of each 128-bit subvector of x and y. +// // Asm: VPUNPCKHWD, CPU Feature: AVX2 func (x Uint16x16) InterleaveHiGrouped(y Uint16x16) Uint16x16 @@ -3373,6 +3403,11 @@ // InterleaveLo interleaves the elements of the low halves of x and y. // +// Asm: VPUNPCKLBW, CPU Feature: AVX +func (x Int8x16) InterleaveLo(y Int8x16) Int8x16 + +// InterleaveLo interleaves the elements of the low halves of x and y. +// // Asm: VPUNPCKLWD, CPU Feature: AVX func (x Int16x8) InterleaveLo(y Int16x8) Int16x8 @@ -3388,6 +3423,11 @@ // InterleaveLo interleaves the elements of the low halves of x and y. // +// Asm: VPUNPCKLBW, CPU Feature: AVX +func (x Uint8x16) InterleaveLo(y Uint8x16) Uint8x16 + +// InterleaveLo interleaves the elements of the low halves of x and y. +// // Asm: VPUNPCKLWD, CPU Feature: AVX func (x Uint16x8) InterleaveLo(y Uint16x8) Uint16x8 @@ -3405,6 +3445,16 @@ // InterleaveLoGrouped interleaves the elements of the low half of each 128-bit subvector of x and y. // +// Asm: VPUNPCKLBW, CPU Feature: AVX2 +func (x Int8x32) InterleaveLoGrouped(y Int8x32) Int8x32 + +// InterleaveLoGrouped interleaves the elements of the low half of each 128-bit subvector of x and y. +// +// Asm: VPUNPCKLBW, CPU Feature: AVX512 +func (x Int8x64) InterleaveLoGrouped(y Int8x64) Int8x64 + +// InterleaveLoGrouped interleaves the elements of the low half of each 128-bit subvector of x and y. +// // Asm: VPUNPCKLWD, CPU Feature: AVX2 func (x Int16x16) InterleaveLoGrouped(y Int16x16) Int16x16 @@ -3435,6 +3485,16 @@ // InterleaveLoGrouped interleaves the elements of the low half of each 128-bit subvector of x and y. // +// Asm: VPUNPCKLBW, CPU Feature: AVX2 +func (x Uint8x32) InterleaveLoGrouped(y Uint8x32) Uint8x32 + +// InterleaveLoGrouped interleaves the elements of the low half of each 128-bit subvector of x and y. +// +// Asm: VPUNPCKLBW, CPU Feature: AVX512 +func (x Uint8x64) InterleaveLoGrouped(y Uint8x64) Uint8x64 + +// InterleaveLoGrouped interleaves the elements of the low half of each 128-bit subvector of x and y. +// // Asm: VPUNPCKLWD, CPU Feature: AVX2 func (x Uint16x16) InterleaveLoGrouped(y Uint16x16) Uint16x16