| // Copyright 2024 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| package main |
| |
| import ( |
| . "github.com/mmcloughlin/avo/build" |
| "github.com/mmcloughlin/avo/ir" |
| . "github.com/mmcloughlin/avo/operand" |
| . "github.com/mmcloughlin/avo/reg" |
| _ "golang.org/x/crypto/blake2b" |
| ) |
| |
| //go:generate go run . -out ../../blake2bAVX2_amd64.s -pkg blake2b |
| |
| const ThatPeskyUnicodeDot = "\u00b7" |
| |
| func main() { |
| Package("golang.org/x/crypto/blake2b") |
| ConstraintExpr("amd64,gc,!purego") |
| hashBlocksAVX2() |
| hashBlocksAVX() |
| Generate() |
| } |
| |
| // Utility function to emit a BYTE instruction |
| func BYTE(imm Op) { |
| Instruction(&ir.Instruction{Opcode: "BYTE", Operands: []Op{imm}}) |
| } |
| |
| func VPERMQ_0x39_Y1_Y1() { |
| BYTE(U8(0xc4)) |
| BYTE(U8(0xe3)) |
| BYTE(U8(0xfd)) |
| BYTE(U8(0x00)) |
| BYTE(U8(0xc9)) |
| BYTE(U8(0x39)) |
| } |
| |
| func VPERMQ_0x93_Y1_Y1() { |
| BYTE(U8(0xc4)) |
| BYTE(U8(0xe3)) |
| BYTE(U8(0xfd)) |
| BYTE(U8(0x00)) |
| BYTE(U8(0xc9)) |
| BYTE(U8(0x93)) |
| } |
| |
| func VPERMQ_0x4E_Y2_Y2() { |
| BYTE(U8(0xc4)) |
| BYTE(U8(0xe3)) |
| BYTE(U8(0xfd)) |
| BYTE(U8(0x00)) |
| BYTE(U8(0xd2)) |
| BYTE(U8(0x4e)) |
| } |
| |
| func VPERMQ_0x93_Y3_Y3() { |
| BYTE(U8(0xc4)) |
| BYTE(U8(0xe3)) |
| BYTE(U8(0xfd)) |
| BYTE(U8(0x00)) |
| BYTE(U8(0xdb)) |
| BYTE(U8(0x93)) |
| } |
| |
| func VPERMQ_0x39_Y3_Y3() { |
| BYTE(U8(0xc4)) |
| BYTE(U8(0xe3)) |
| BYTE(U8(0xfd)) |
| BYTE(U8(0x00)) |
| BYTE(U8(0xdb)) |
| BYTE(U8(0x39)) |
| } |
| |
| func ROUND_AVX2(m0, m1, m2, m3 Op, t, c40, c48 VecPhysical) { |
| VPADDQ(m0, Y0, Y0) |
| VPADDQ(Y1, Y0, Y0) |
| VPXOR(Y0, Y3, Y3) |
| VPSHUFD(I8(-79), Y3, Y3) |
| VPADDQ(Y3, Y2, Y2) |
| VPXOR(Y2, Y1, Y1) |
| VPSHUFB(c40, Y1, Y1) |
| VPADDQ(m1, Y0, Y0) |
| VPADDQ(Y1, Y0, Y0) |
| VPXOR(Y0, Y3, Y3) |
| VPSHUFB(c48, Y3, Y3) |
| VPADDQ(Y3, Y2, Y2) |
| VPXOR(Y2, Y1, Y1) |
| VPADDQ(Y1, Y1, t) |
| VPSRLQ(Imm(63), Y1, Y1) |
| VPXOR(t, Y1, Y1) |
| VPERMQ_0x39_Y1_Y1() |
| VPERMQ_0x4E_Y2_Y2() |
| VPERMQ_0x93_Y3_Y3() |
| VPADDQ(m2, Y0, Y0) |
| VPADDQ(Y1, Y0, Y0) |
| VPXOR(Y0, Y3, Y3) |
| VPSHUFD(I8(-79), Y3, Y3) |
| VPADDQ(Y3, Y2, Y2) |
| VPXOR(Y2, Y1, Y1) |
| VPSHUFB(c40, Y1, Y1) |
| VPADDQ(m3, Y0, Y0) |
| VPADDQ(Y1, Y0, Y0) |
| VPXOR(Y0, Y3, Y3) |
| VPSHUFB(c48, Y3, Y3) |
| VPADDQ(Y3, Y2, Y2) |
| VPXOR(Y2, Y1, Y1) |
| VPADDQ(Y1, Y1, t) |
| VPSRLQ(Imm(63), Y1, Y1) |
| VPXOR(t, Y1, Y1) |
| VPERMQ_0x39_Y3_Y3() |
| VPERMQ_0x4E_Y2_Y2() |
| VPERMQ_0x93_Y1_Y1() |
| } |
| |
| func VMOVQ_SI_X11_0() { |
| BYTE(U8(0xC5)) |
| BYTE(U8(0x7A)) |
| BYTE(U8(0x7E)) |
| BYTE(U8(0x1E)) |
| } |
| |
| func VMOVQ_SI_X12_0() { |
| BYTE(U8(0xC5)) |
| BYTE(U8(0x7A)) |
| BYTE(U8(0x7E)) |
| BYTE(U8(0x26)) |
| } |
| |
| func VMOVQ_SI_X13_0() { |
| BYTE(U8(0xC5)) |
| BYTE(U8(0x7A)) |
| BYTE(U8(0x7E)) |
| BYTE(U8(0x2E)) |
| } |
| |
| func VMOVQ_SI_X14_0() { |
| BYTE(U8(0xC5)) |
| BYTE(U8(0x7A)) |
| BYTE(U8(0x7E)) |
| BYTE(U8(0x36)) |
| } |
| |
| func VMOVQ_SI_X15_0() { |
| BYTE(U8(0xC5)) |
| BYTE(U8(0x7A)) |
| BYTE(U8(0x7E)) |
| BYTE(U8(0x3E)) |
| } |
| |
| func VMOVQ_SI_X11(n uint8) { |
| BYTE(U8(0xC5)) |
| BYTE(U8(0x7A)) |
| BYTE(U8(0x7E)) |
| BYTE(U8(0x5E)) |
| BYTE(U8(n)) |
| } |
| |
| func VMOVQ_SI_X12(n uint8) { |
| BYTE(U8(0xC5)) |
| BYTE(U8(0x7A)) |
| BYTE(U8(0x7E)) |
| BYTE(U8(0x66)) |
| BYTE(U8(n)) |
| } |
| |
| func VMOVQ_SI_X13(n uint8) { |
| BYTE(U8(0xC5)) |
| BYTE(U8(0x7A)) |
| BYTE(U8(0x7E)) |
| BYTE(U8(0x6E)) |
| BYTE(U8(n)) |
| } |
| |
| func VMOVQ_SI_X14(n uint8) { |
| BYTE(U8(0xC5)) |
| BYTE(U8(0x7A)) |
| BYTE(U8(0x7E)) |
| BYTE(U8(0x76)) |
| BYTE(U8(n)) |
| } |
| |
| func VMOVQ_SI_X15(n uint8) { |
| BYTE(U8(0xC5)) |
| BYTE(U8(0x7A)) |
| BYTE(U8(0x7E)) |
| BYTE(U8(0x7E)) |
| BYTE(U8(n)) |
| } |
| |
| func VPINSRQ_1_SI_X11_0() { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0x63)) |
| BYTE(U8(0xA1)) |
| BYTE(U8(0x22)) |
| BYTE(U8(0x1E)) |
| BYTE(U8(0x01)) |
| } |
| |
| func VPINSRQ_1_SI_X12_0() { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0x63)) |
| BYTE(U8(0x99)) |
| BYTE(U8(0x22)) |
| BYTE(U8(0x26)) |
| BYTE(U8(0x01)) |
| } |
| |
| func VPINSRQ_1_SI_X13_0() { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0x63)) |
| BYTE(U8(0x91)) |
| BYTE(U8(0x22)) |
| BYTE(U8(0x2E)) |
| BYTE(U8(0x01)) |
| } |
| |
| func VPINSRQ_1_SI_X14_0() { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0x63)) |
| BYTE(U8(0x89)) |
| BYTE(U8(0x22)) |
| BYTE(U8(0x36)) |
| BYTE(U8(0x01)) |
| } |
| |
| func VPINSRQ_1_SI_X15_0() { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0x63)) |
| BYTE(U8(0x81)) |
| BYTE(U8(0x22)) |
| BYTE(U8(0x3E)) |
| BYTE(U8(0x01)) |
| } |
| |
| func VPINSRQ_1_SI_X11(n uint8) { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0x63)) |
| BYTE(U8(0xA1)) |
| BYTE(U8(0x22)) |
| BYTE(U8(0x5E)) |
| BYTE(U8(n)) |
| BYTE(U8(0x01)) |
| } |
| |
| func VPINSRQ_1_SI_X12(n uint8) { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0x63)) |
| BYTE(U8(0x99)) |
| BYTE(U8(0x22)) |
| BYTE(U8(0x66)) |
| BYTE(U8(n)) |
| BYTE(U8(0x01)) |
| } |
| |
| func VPINSRQ_1_SI_X13(n uint8) { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0x63)) |
| BYTE(U8(0x91)) |
| BYTE(U8(0x22)) |
| BYTE(U8(0x6E)) |
| BYTE(U8(n)) |
| BYTE(U8(0x01)) |
| } |
| |
| func VPINSRQ_1_SI_X14(n uint8) { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0x63)) |
| BYTE(U8(0x89)) |
| BYTE(U8(0x22)) |
| BYTE(U8(0x76)) |
| BYTE(U8(n)) |
| BYTE(U8(0x01)) |
| } |
| |
| func VPINSRQ_1_SI_X15(n uint8) { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0x63)) |
| BYTE(U8(0x81)) |
| BYTE(U8(0x22)) |
| BYTE(U8(0x7E)) |
| BYTE(U8(n)) |
| BYTE(U8(0x01)) |
| } |
| |
| func VMOVQ_R8_X15() { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0x41)) |
| BYTE(U8(0xF9)) |
| BYTE(U8(0x6E)) |
| BYTE(U8(0xF8)) |
| } |
| |
| func VPINSRQ_1_R9_X15() { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0x43)) |
| BYTE(U8(0x81)) |
| BYTE(U8(0x22)) |
| BYTE(U8(0xF9)) |
| BYTE(U8(0x01)) |
| } |
| |
| // load msg: |
| // |
| // Y12 = (i0, i1, i2, i3) |
| // |
| // i0, i1, i2, i3 must not be 0 |
| func LOAD_MSG_AVX2_Y12(i0, i1, i2, i3 uint8) { |
| VMOVQ_SI_X12(i0 * 8) |
| VMOVQ_SI_X11(i2 * 8) |
| VPINSRQ_1_SI_X12(i1 * 8) |
| VPINSRQ_1_SI_X11(i3 * 8) |
| VINSERTI128(Imm(1), X11, Y12, Y12) |
| } |
| |
| // load msg: |
| // |
| // Y13 = (i0, i1, i2, i3) |
| // |
| // i0, i1, i2, i3 must not be 0 |
| func LOAD_MSG_AVX2_Y13(i0, i1, i2, i3 uint8) { |
| VMOVQ_SI_X13(i0 * 8) |
| VMOVQ_SI_X11(i2 * 8) |
| VPINSRQ_1_SI_X13(i1 * 8) |
| VPINSRQ_1_SI_X11(i3 * 8) |
| VINSERTI128(Imm(1), X11, Y13, Y13) |
| } |
| |
| // load msg: |
| // |
| // Y14 = (i0, i1, i2, i3) |
| // |
| // i0, i1, i2, i3 must not be 0 |
| func LOAD_MSG_AVX2_Y14(i0, i1, i2, i3 uint8) { |
| VMOVQ_SI_X14(i0 * 8) |
| VMOVQ_SI_X11(i2 * 8) |
| VPINSRQ_1_SI_X14(i1 * 8) |
| VPINSRQ_1_SI_X11(i3 * 8) |
| VINSERTI128(Imm(1), X11, Y14, Y14) |
| } |
| |
| // load msg: |
| // |
| // Y15 = (i0, i1, i2, i3) |
| // |
| // i0, i1, i2, i3 must not be 0 |
| func LOAD_MSG_AVX2_Y15(i0, i1, i2, i3 uint8) { |
| VMOVQ_SI_X15(i0 * 8) |
| VMOVQ_SI_X11(i2 * 8) |
| VPINSRQ_1_SI_X15(i1 * 8) |
| VPINSRQ_1_SI_X11(i3 * 8) |
| VINSERTI128(Imm(1), X11, Y15, Y15) |
| } |
| |
| func LOAD_MSG_AVX2_0_2_4_6_1_3_5_7_8_10_12_14_9_11_13_15() { |
| VMOVQ_SI_X12_0() |
| VMOVQ_SI_X11(4 * 8) |
| VPINSRQ_1_SI_X12(2 * 8) |
| VPINSRQ_1_SI_X11(6 * 8) |
| VINSERTI128(Imm(1), X11, Y12, Y12) |
| LOAD_MSG_AVX2_Y13(1, 3, 5, 7) |
| LOAD_MSG_AVX2_Y14(8, 10, 12, 14) |
| LOAD_MSG_AVX2_Y15(9, 11, 13, 15) |
| } |
| |
| func LOAD_MSG_AVX2_14_4_9_13_10_8_15_6_1_0_11_5_12_2_7_3() { |
| LOAD_MSG_AVX2_Y12(14, 4, 9, 13) |
| LOAD_MSG_AVX2_Y13(10, 8, 15, 6) |
| VMOVQ_SI_X11(11 * 8) |
| VPSHUFD(Imm(0x4E), Mem{Base: SI}.Offset(0*8), X14) |
| VPINSRQ_1_SI_X11(5 * 8) |
| VINSERTI128(Imm(1), X11, Y14, Y14) |
| LOAD_MSG_AVX2_Y15(12, 2, 7, 3) |
| } |
| |
| func LOAD_MSG_AVX2_11_12_5_15_8_0_2_13_10_3_7_9_14_6_1_4() { |
| VMOVQ_SI_X11(5 * 8) |
| VMOVDQU(Mem{Base: SI}.Offset(11*8), X12) |
| VPINSRQ_1_SI_X11(15 * 8) |
| VINSERTI128(Imm(1), X11, Y12, Y12) |
| VMOVQ_SI_X13(8 * 8) |
| VMOVQ_SI_X11(2 * 8) |
| VPINSRQ_1_SI_X13_0() |
| VPINSRQ_1_SI_X11(13 * 8) |
| VINSERTI128(Imm(1), X11, Y13, Y13) |
| LOAD_MSG_AVX2_Y14(10, 3, 7, 9) |
| LOAD_MSG_AVX2_Y15(14, 6, 1, 4) |
| } |
| |
| func LOAD_MSG_AVX2_7_3_13_11_9_1_12_14_2_5_4_15_6_10_0_8() { |
| LOAD_MSG_AVX2_Y12(7, 3, 13, 11) |
| LOAD_MSG_AVX2_Y13(9, 1, 12, 14) |
| LOAD_MSG_AVX2_Y14(2, 5, 4, 15) |
| VMOVQ_SI_X15(6 * 8) |
| VMOVQ_SI_X11_0() |
| VPINSRQ_1_SI_X15(10 * 8) |
| VPINSRQ_1_SI_X11(8 * 8) |
| VINSERTI128(Imm(1), X11, Y15, Y15) |
| } |
| |
| func LOAD_MSG_AVX2_9_5_2_10_0_7_4_15_14_11_6_3_1_12_8_13() { |
| LOAD_MSG_AVX2_Y12(9, 5, 2, 10) |
| VMOVQ_SI_X13_0() |
| VMOVQ_SI_X11(4 * 8) |
| VPINSRQ_1_SI_X13(7 * 8) |
| VPINSRQ_1_SI_X11(15 * 8) |
| VINSERTI128(Imm(1), X11, Y13, Y13) |
| LOAD_MSG_AVX2_Y14(14, 11, 6, 3) |
| LOAD_MSG_AVX2_Y15(1, 12, 8, 13) |
| } |
| |
| func LOAD_MSG_AVX2_2_6_0_8_12_10_11_3_4_7_15_1_13_5_14_9() { |
| VMOVQ_SI_X12(2 * 8) |
| VMOVQ_SI_X11_0() |
| VPINSRQ_1_SI_X12(6 * 8) |
| VPINSRQ_1_SI_X11(8 * 8) |
| VINSERTI128(Imm(1), X11, Y12, Y12) |
| LOAD_MSG_AVX2_Y13(12, 10, 11, 3) |
| LOAD_MSG_AVX2_Y14(4, 7, 15, 1) |
| LOAD_MSG_AVX2_Y15(13, 5, 14, 9) |
| } |
| |
| func LOAD_MSG_AVX2_12_1_14_4_5_15_13_10_0_6_9_8_7_3_2_11() { |
| LOAD_MSG_AVX2_Y12(12, 1, 14, 4) |
| LOAD_MSG_AVX2_Y13(5, 15, 13, 10) |
| VMOVQ_SI_X14_0() |
| VPSHUFD(Imm(0x4E), Mem{Base: SI}.Offset(8*8), X11) |
| VPINSRQ_1_SI_X14(6 * 8) |
| VINSERTI128(Imm(1), X11, Y14, Y14) |
| LOAD_MSG_AVX2_Y15(7, 3, 2, 11) |
| } |
| |
| func LOAD_MSG_AVX2_13_7_12_3_11_14_1_9_5_15_8_2_0_4_6_10() { |
| LOAD_MSG_AVX2_Y12(13, 7, 12, 3) |
| LOAD_MSG_AVX2_Y13(11, 14, 1, 9) |
| LOAD_MSG_AVX2_Y14(5, 15, 8, 2) |
| VMOVQ_SI_X15_0() |
| VMOVQ_SI_X11(6 * 8) |
| VPINSRQ_1_SI_X15(4 * 8) |
| VPINSRQ_1_SI_X11(10 * 8) |
| VINSERTI128(Imm(1), X11, Y15, Y15) |
| } |
| |
| func LOAD_MSG_AVX2_6_14_11_0_15_9_3_8_12_13_1_10_2_7_4_5() { |
| VMOVQ_SI_X12(6 * 8) |
| VMOVQ_SI_X11(11 * 8) |
| VPINSRQ_1_SI_X12(14 * 8) |
| VPINSRQ_1_SI_X11_0() |
| VINSERTI128(Imm(1), X11, Y12, Y12) |
| LOAD_MSG_AVX2_Y13(15, 9, 3, 8) |
| VMOVQ_SI_X11(1 * 8) |
| VMOVDQU(Mem{Base: SI}.Offset(12*8), X14) |
| VPINSRQ_1_SI_X11(10 * 8) |
| VINSERTI128(Imm(1), X11, Y14, Y14) |
| VMOVQ_SI_X15(2 * 8) |
| VMOVDQU(Mem{Base: SI}.Offset(4*8), X11) |
| VPINSRQ_1_SI_X15(7 * 8) |
| VINSERTI128(Imm(1), X11, Y15, Y15) |
| } |
| |
| func LOAD_MSG_AVX2_10_8_7_1_2_4_6_5_15_9_3_13_11_14_12_0() { |
| LOAD_MSG_AVX2_Y12(10, 8, 7, 1) |
| VMOVQ_SI_X13(2 * 8) |
| VPSHUFD(Imm(0x4E), Mem{Base: SI}.Offset(5*8), X11) |
| VPINSRQ_1_SI_X13(4 * 8) |
| VINSERTI128(Imm(1), X11, Y13, Y13) |
| LOAD_MSG_AVX2_Y14(15, 9, 3, 13) |
| VMOVQ_SI_X15(11 * 8) |
| VMOVQ_SI_X11(12 * 8) |
| VPINSRQ_1_SI_X15(14 * 8) |
| VPINSRQ_1_SI_X11_0() |
| VINSERTI128(Imm(1), X11, Y15, Y15) |
| } |
| |
| func hashBlocksAVX2() { |
| Implement("hashBlocksAVX2") |
| Attributes(4) |
| AllocLocal(320) // frame size = 288 + 32 byte alignment |
| |
| Load(Param("h"), RAX) |
| Load(Param("c"), RBX) |
| Load(Param("flag"), RCX) |
| Load(Param("blocks").Base(), RSI) |
| Load(Param("blocks").Len(), RDI) |
| |
| MOVQ(RSP, RDX) |
| ADDQ(I32(31), RDX) |
| ANDQ(I32(^31), RDX) |
| |
| MOVQ(RCX, Mem{Base: DX}.Offset(16)) |
| XORQ(RCX, RCX) |
| MOVQ(RCX, Mem{Base: DX}.Offset(24)) |
| |
| AVX2_c40 := AVX2_c40_DATA() |
| AVX2_c48 := AVX2_c48_DATA() |
| VMOVDQU(AVX2_c40, Y4) |
| VMOVDQU(AVX2_c48, Y5) |
| |
| VMOVDQU(Mem{Base: AX}.Offset(0), Y8) |
| VMOVDQU(Mem{Base: AX}.Offset(32), Y9) |
| AVX2_iv0 := AVX2_iv0_DATA() |
| AVX2_iv1 := AVX2_iv1_DATA() |
| VMOVDQU(AVX2_iv0, Y6) |
| VMOVDQU(AVX2_iv1, Y7) |
| |
| MOVQ(Mem{Base: BX}.Offset(0), R8) |
| MOVQ(Mem{Base: BX}.Offset(8), R9) |
| MOVQ(R9, Mem{Base: DX}.Offset(8)) |
| |
| loop_AVX2() |
| noinc_AVX2() |
| } |
| |
| func loop_AVX2() { |
| Label("loop") |
| ADDQ(Imm(128), R8) |
| MOVQ(R8, Mem{Base: DX}.Offset(0)) |
| CMPQ(R8, Imm(128)) |
| JGE(LabelRef("noinc")) |
| INCQ(R9) |
| MOVQ(R9, Mem{Base: DX}.Offset(8)) |
| } |
| |
| // line 312 |
| func noinc_AVX2() { |
| Label("noinc") |
| VMOVDQA(Y8, Y0) |
| VMOVDQA(Y9, Y1) |
| VMOVDQA(Y6, Y2) |
| VPXOR(Mem{Base: DX}.Offset(0), Y7, Y3) |
| |
| LOAD_MSG_AVX2_0_2_4_6_1_3_5_7_8_10_12_14_9_11_13_15() |
| VMOVDQA(Y12, Mem{Base: DX}.Offset(32)) |
| VMOVDQA(Y13, Mem{Base: DX}.Offset(64)) |
| VMOVDQA(Y14, Mem{Base: DX}.Offset(96)) |
| VMOVDQA(Y15, Mem{Base: DX}.Offset(128)) |
| ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5) |
| LOAD_MSG_AVX2_14_4_9_13_10_8_15_6_1_0_11_5_12_2_7_3() |
| VMOVDQA(Y12, Mem{Base: DX}.Offset(160)) |
| VMOVDQA(Y13, Mem{Base: DX}.Offset(192)) |
| VMOVDQA(Y14, Mem{Base: DX}.Offset(224)) |
| VMOVDQA(Y15, Mem{Base: DX}.Offset(256)) |
| |
| ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5) |
| LOAD_MSG_AVX2_11_12_5_15_8_0_2_13_10_3_7_9_14_6_1_4() |
| ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5) |
| LOAD_MSG_AVX2_7_3_13_11_9_1_12_14_2_5_4_15_6_10_0_8() |
| ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5) |
| LOAD_MSG_AVX2_9_5_2_10_0_7_4_15_14_11_6_3_1_12_8_13() |
| ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5) |
| LOAD_MSG_AVX2_2_6_0_8_12_10_11_3_4_7_15_1_13_5_14_9() |
| ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5) |
| LOAD_MSG_AVX2_12_1_14_4_5_15_13_10_0_6_9_8_7_3_2_11() |
| ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5) |
| LOAD_MSG_AVX2_13_7_12_3_11_14_1_9_5_15_8_2_0_4_6_10() |
| ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5) |
| LOAD_MSG_AVX2_6_14_11_0_15_9_3_8_12_13_1_10_2_7_4_5() |
| ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5) |
| LOAD_MSG_AVX2_10_8_7_1_2_4_6_5_15_9_3_13_11_14_12_0() |
| ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5) |
| |
| ROUND_AVX2(Mem{Base: DX}.Offset(32), Mem{Base: DX}.Offset(64), Mem{Base: DX}.Offset(96), Mem{Base: DX}.Offset(128), Y10, Y4, Y5) |
| ROUND_AVX2(Mem{Base: DX}.Offset(160), Mem{Base: DX}.Offset(192), Mem{Base: DX}.Offset(224), Mem{Base: DX}.Offset(256), Y10, Y4, Y5) |
| |
| VPXOR(Y0, Y8, Y8) |
| VPXOR(Y1, Y9, Y9) |
| VPXOR(Y2, Y8, Y8) |
| VPXOR(Y3, Y9, Y9) |
| |
| LEAQ(Mem{Base: SI}.Offset(128), RSI) |
| SUBQ(Imm(128), RDI) |
| JNE(LabelRef("loop")) |
| |
| MOVQ(R8, Mem{Base: BX}.Offset(0)) |
| MOVQ(R9, Mem{Base: BX}.Offset(8)) |
| |
| VMOVDQU(Y8, Mem{Base: AX}.Offset(0)) |
| VMOVDQU(Y9, Mem{Base: AX}.Offset(32)) |
| VZEROUPPER() |
| |
| RET() |
| } |
| |
| func VPUNPCKLQDQ_X2_X2_X15() { |
| BYTE(U8(0xC5)) |
| BYTE(U8(0x69)) |
| BYTE(U8(0x6C)) |
| BYTE(U8(0xFA)) |
| } |
| |
| func VPUNPCKLQDQ_X3_X3_X15() { |
| BYTE(U8(0xC5)) |
| BYTE(U8(0x61)) |
| BYTE(U8(0x6C)) |
| BYTE(U8(0xFB)) |
| } |
| |
| func VPUNPCKLQDQ_X7_X7_X15() { |
| BYTE(U8(0xC5)) |
| BYTE(U8(0x41)) |
| BYTE(U8(0x6C)) |
| BYTE(U8(0xFF)) |
| } |
| |
| func VPUNPCKLQDQ_X13_X13_X15() { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0x41)) |
| BYTE(U8(0x11)) |
| BYTE(U8(0x6C)) |
| BYTE(U8(0xFD)) |
| } |
| |
| func VPUNPCKLQDQ_X14_X14_X15() { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0x41)) |
| BYTE(U8(0x09)) |
| BYTE(U8(0x6C)) |
| BYTE(U8(0xFE)) |
| } |
| |
| func VPUNPCKHQDQ_X15_X2_X2() { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0xC1)) |
| BYTE(U8(0x69)) |
| BYTE(U8(0x6D)) |
| BYTE(U8(0xD7)) |
| } |
| |
| func VPUNPCKHQDQ_X15_X3_X3() { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0xC1)) |
| BYTE(U8(0x61)) |
| BYTE(U8(0x6D)) |
| BYTE(U8(0xDF)) |
| } |
| |
| func VPUNPCKHQDQ_X15_X6_X6() { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0xC1)) |
| BYTE(U8(0x49)) |
| BYTE(U8(0x6D)) |
| BYTE(U8(0xF7)) |
| } |
| |
| func VPUNPCKHQDQ_X15_X7_X7() { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0xC1)) |
| BYTE(U8(0x41)) |
| BYTE(U8(0x6D)) |
| BYTE(U8(0xFF)) |
| } |
| |
| func VPUNPCKHQDQ_X15_X3_X2() { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0xC1)) |
| BYTE(U8(0x61)) |
| BYTE(U8(0x6D)) |
| BYTE(U8(0xD7)) |
| } |
| |
| func VPUNPCKHQDQ_X15_X7_X6() { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0xC1)) |
| BYTE(U8(0x41)) |
| BYTE(U8(0x6D)) |
| BYTE(U8(0xF7)) |
| } |
| |
| func VPUNPCKHQDQ_X15_X13_X3() { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0xC1)) |
| BYTE(U8(0x11)) |
| BYTE(U8(0x6D)) |
| BYTE(U8(0xDF)) |
| } |
| |
| func VPUNPCKHQDQ_X15_X13_X7() { |
| BYTE(U8(0xC4)) |
| BYTE(U8(0xC1)) |
| BYTE(U8(0x11)) |
| BYTE(U8(0x6D)) |
| BYTE(U8(0xFF)) |
| } |
| |
| func SHUFFLE_AVX() { |
| VMOVDQA(X6, X13) |
| VMOVDQA(X2, X14) |
| VMOVDQA(X4, X6) |
| VPUNPCKLQDQ_X13_X13_X15() |
| VMOVDQA(X5, X4) |
| VMOVDQA(X6, X5) |
| VPUNPCKHQDQ_X15_X7_X6() |
| VPUNPCKLQDQ_X7_X7_X15() |
| VPUNPCKHQDQ_X15_X13_X7() |
| VPUNPCKLQDQ_X3_X3_X15() |
| VPUNPCKHQDQ_X15_X2_X2() |
| VPUNPCKLQDQ_X14_X14_X15() |
| VPUNPCKHQDQ_X15_X3_X3() |
| } |
| |
| func SHUFFLE_AVX_INV() { |
| VMOVDQA(X2, X13) |
| VMOVDQA(X4, X14) |
| VPUNPCKLQDQ_X2_X2_X15() |
| VMOVDQA(X5, X4) |
| VPUNPCKHQDQ_X15_X3_X2() |
| VMOVDQA(X14, X5) |
| VPUNPCKLQDQ_X3_X3_X15() |
| VMOVDQA(X6, X14) |
| VPUNPCKHQDQ_X15_X13_X3() |
| VPUNPCKLQDQ_X7_X7_X15() |
| VPUNPCKHQDQ_X15_X6_X6() |
| VPUNPCKLQDQ_X14_X14_X15() |
| VPUNPCKHQDQ_X15_X7_X7() |
| } |
| |
| func HALF_ROUND_AVX(v0, v1, v2, v3, v4, v5, v6, v7 VecPhysical, m0, m1, m2, m3 Op, t0, c40, c48 VecPhysical) { |
| VPADDQ(m0, v0, v0) |
| VPADDQ(v2, v0, v0) |
| VPADDQ(m1, v1, v1) |
| VPADDQ(v3, v1, v1) |
| VPXOR(v0, v6, v6) |
| VPXOR(v1, v7, v7) |
| VPSHUFD(I8(-79), v6, v6) |
| VPSHUFD(I8(-79), v7, v7) |
| VPADDQ(v6, v4, v4) |
| VPADDQ(v7, v5, v5) |
| VPXOR(v4, v2, v2) |
| VPXOR(v5, v3, v3) |
| VPSHUFB(c40, v2, v2) |
| VPSHUFB(c40, v3, v3) |
| VPADDQ(m2, v0, v0) |
| VPADDQ(v2, v0, v0) |
| VPADDQ(m3, v1, v1) |
| VPADDQ(v3, v1, v1) |
| VPXOR(v0, v6, v6) |
| VPXOR(v1, v7, v7) |
| VPSHUFB(c48, v6, v6) |
| VPSHUFB(c48, v7, v7) |
| VPADDQ(v6, v4, v4) |
| VPADDQ(v7, v5, v5) |
| VPXOR(v4, v2, v2) |
| VPXOR(v5, v3, v3) |
| VPADDQ(v2, v2, t0) |
| VPSRLQ(Imm(63), v2, v2) |
| VPXOR(t0, v2, v2) |
| VPADDQ(v3, v3, t0) |
| VPSRLQ(Imm(63), v3, v3) |
| VPXOR(t0, v3, v3) |
| } |
| |
| // load msg: |
| // |
| // X12 = (i0, i1), X13 = (i2, i3), X14 = (i4, i5), X15 = (i6, i7) |
| // |
| // i0, i1, i2, i3, i4, i5, i6, i7 must not be 0 |
| func LOAD_MSG_AVX(i0, i1, i2, i3, i4, i5, i6, i7 uint8) { |
| VMOVQ_SI_X12(i0 * 8) |
| VMOVQ_SI_X13(i2 * 8) |
| VMOVQ_SI_X14(i4 * 8) |
| VMOVQ_SI_X15(i6 * 8) |
| VPINSRQ_1_SI_X12(i1 * 8) |
| VPINSRQ_1_SI_X13(i3 * 8) |
| VPINSRQ_1_SI_X14(i5 * 8) |
| VPINSRQ_1_SI_X15(i7 * 8) |
| } |
| |
| // load msg: |
| // |
| // X12 = (0, 2), X13 = (4, 6), X14 = (1, 3), X15 = (5, 7) |
| func LOAD_MSG_AVX_0_2_4_6_1_3_5_7() { |
| VMOVQ_SI_X12_0() |
| VMOVQ_SI_X13(4 * 8) |
| VMOVQ_SI_X14(1 * 8) |
| VMOVQ_SI_X15(5 * 8) |
| VPINSRQ_1_SI_X12(2 * 8) |
| VPINSRQ_1_SI_X13(6 * 8) |
| VPINSRQ_1_SI_X14(3 * 8) |
| VPINSRQ_1_SI_X15(7 * 8) |
| } |
| |
| // load msg: |
| // |
| // X12 = (1, 0), X13 = (11, 5), X14 = (12, 2), X15 = (7, 3) |
| func LOAD_MSG_AVX_1_0_11_5_12_2_7_3() { |
| VPSHUFD(Imm(0x4E), Mem{Base: SI}.Offset(0*8), X12) |
| VMOVQ_SI_X13(11 * 8) |
| VMOVQ_SI_X14(12 * 8) |
| VMOVQ_SI_X15(7 * 8) |
| VPINSRQ_1_SI_X13(5 * 8) |
| VPINSRQ_1_SI_X14(2 * 8) |
| VPINSRQ_1_SI_X15(3 * 8) |
| } |
| |
| // load msg: |
| // |
| // X12 = (11, 12), X13 = (5, 15), X14 = (8, 0), X15 = (2, 13) |
| func LOAD_MSG_AVX_11_12_5_15_8_0_2_13() { |
| VMOVDQU(Mem{Base: SI}.Offset(11*8), X12) |
| VMOVQ_SI_X13(5 * 8) |
| VMOVQ_SI_X14(8 * 8) |
| VMOVQ_SI_X15(2 * 8) |
| VPINSRQ_1_SI_X13(15 * 8) |
| VPINSRQ_1_SI_X14_0() |
| VPINSRQ_1_SI_X15(13 * 8) |
| } |
| |
| // load msg: |
| // |
| // X12 = (2, 5), X13 = (4, 15), X14 = (6, 10), X15 = (0, 8) |
| func LOAD_MSG_AVX_2_5_4_15_6_10_0_8() { |
| VMOVQ_SI_X12(2 * 8) |
| VMOVQ_SI_X13(4 * 8) |
| VMOVQ_SI_X14(6 * 8) |
| VMOVQ_SI_X15_0() |
| VPINSRQ_1_SI_X12(5 * 8) |
| VPINSRQ_1_SI_X13(15 * 8) |
| VPINSRQ_1_SI_X14(10 * 8) |
| VPINSRQ_1_SI_X15(8 * 8) |
| } |
| |
| // load msg: |
| // |
| // X12 = (9, 5), X13 = (2, 10), X14 = (0, 7), X15 = (4, 15) |
| func LOAD_MSG_AVX_9_5_2_10_0_7_4_15() { |
| VMOVQ_SI_X12(9 * 8) |
| VMOVQ_SI_X13(2 * 8) |
| VMOVQ_SI_X14_0() |
| VMOVQ_SI_X15(4 * 8) |
| VPINSRQ_1_SI_X12(5 * 8) |
| VPINSRQ_1_SI_X13(10 * 8) |
| VPINSRQ_1_SI_X14(7 * 8) |
| VPINSRQ_1_SI_X15(15 * 8) |
| } |
| |
| // load msg: |
| // |
| // X12 = (2, 6), X13 = (0, 8), X14 = (12, 10), X15 = (11, 3) |
| func LOAD_MSG_AVX_2_6_0_8_12_10_11_3() { |
| VMOVQ_SI_X12(2 * 8) |
| VMOVQ_SI_X13_0() |
| VMOVQ_SI_X14(12 * 8) |
| VMOVQ_SI_X15(11 * 8) |
| VPINSRQ_1_SI_X12(6 * 8) |
| VPINSRQ_1_SI_X13(8 * 8) |
| VPINSRQ_1_SI_X14(10 * 8) |
| VPINSRQ_1_SI_X15(3 * 8) |
| } |
| |
| // load msg: |
| // |
| // X12 = (0, 6), X13 = (9, 8), X14 = (7, 3), X15 = (2, 11) |
| func LOAD_MSG_AVX_0_6_9_8_7_3_2_11() { |
| MOVQ(Mem{Base: SI}.Offset(0*8), X12) |
| VPSHUFD(Imm(0x4E), Mem{Base: SI}.Offset(8*8), X13) |
| MOVQ(Mem{Base: SI}.Offset(7*8), X14) |
| MOVQ(Mem{Base: SI}.Offset(2*8), X15) |
| VPINSRQ_1_SI_X12(6 * 8) |
| VPINSRQ_1_SI_X14(3 * 8) |
| VPINSRQ_1_SI_X15(11 * 8) |
| } |
| |
| // load msg: |
| // |
| // X12 = (6, 14), X13 = (11, 0), X14 = (15, 9), X15 = (3, 8) |
| func LOAD_MSG_AVX_6_14_11_0_15_9_3_8() { |
| MOVQ(Mem{Base: SI}.Offset(6*8), X12) |
| MOVQ(Mem{Base: SI}.Offset(11*8), X13) |
| MOVQ(Mem{Base: SI}.Offset(15*8), X14) |
| MOVQ(Mem{Base: SI}.Offset(3*8), X15) |
| VPINSRQ_1_SI_X12(14 * 8) |
| VPINSRQ_1_SI_X13_0() |
| VPINSRQ_1_SI_X14(9 * 8) |
| VPINSRQ_1_SI_X15(8 * 8) |
| } |
| |
| // load msg: |
| // |
| // X12 = (5, 15), X13 = (8, 2), X14 = (0, 4), X15 = (6, 10) |
| func LOAD_MSG_AVX_5_15_8_2_0_4_6_10() { |
| MOVQ(Mem{Base: SI}.Offset(5*8), X12) |
| MOVQ(Mem{Base: SI}.Offset(8*8), X13) |
| MOVQ(Mem{Base: SI}.Offset(0*8), X14) |
| MOVQ(Mem{Base: SI}.Offset(6*8), X15) |
| VPINSRQ_1_SI_X12(15 * 8) |
| VPINSRQ_1_SI_X13(2 * 8) |
| VPINSRQ_1_SI_X14(4 * 8) |
| VPINSRQ_1_SI_X15(10 * 8) |
| } |
| |
| // load msg: |
| // |
| // X12 = (12, 13), X13 = (1, 10), X14 = (2, 7), X15 = (4, 5) |
| func LOAD_MSG_AVX_12_13_1_10_2_7_4_5() { |
| VMOVDQU(Mem{Base: SI}.Offset(12*8), X12) |
| MOVQ(Mem{Base: SI}.Offset(1*8), X13) |
| MOVQ(Mem{Base: SI}.Offset(2*8), X14) |
| VPINSRQ_1_SI_X13(10 * 8) |
| VPINSRQ_1_SI_X14(7 * 8) |
| VMOVDQU(Mem{Base: SI}.Offset(4*8), X15) |
| } |
| |
| // load msg: |
| // |
| // X12 = (15, 9), X13 = (3, 13), X14 = (11, 14), X15 = (12, 0) |
| func LOAD_MSG_AVX_15_9_3_13_11_14_12_0() { |
| MOVQ(Mem{Base: SI}.Offset(15*8), X12) |
| MOVQ(Mem{Base: SI}.Offset(3*8), X13) |
| MOVQ(Mem{Base: SI}.Offset(11*8), X14) |
| MOVQ(Mem{Base: SI}.Offset(12*8), X15) |
| VPINSRQ_1_SI_X12(9 * 8) |
| VPINSRQ_1_SI_X13(13 * 8) |
| VPINSRQ_1_SI_X14(14 * 8) |
| VPINSRQ_1_SI_X15_0() |
| } |
| |
| func hashBlocksAVX() { |
| Implement("hashBlocksAVX") |
| Attributes(4) |
| AllocLocal(288) // frame size = 272 + 16 byte alignment |
| |
| Load(Param("h"), RAX) |
| Load(Param("c"), RBX) |
| Load(Param("flag"), RCX) |
| Load(Param("blocks").Base(), RSI) |
| Load(Param("blocks").Len(), RDI) |
| |
| MOVQ(RSP, R10) |
| ADDQ(Imm(15), R10) |
| ANDQ(I32(^15), R10) |
| |
| AVX_c40 := AVX_c40_DATA() |
| AVX_c48 := AVX_c48_DATA() |
| VMOVDQU(AVX_c40, X0) |
| VMOVDQU(AVX_c48, X1) |
| VMOVDQA(X0, X8) |
| VMOVDQA(X1, X9) |
| |
| AVX_iv3 := AVX_iv3_DATA() |
| VMOVDQU(AVX_iv3, X0) |
| VMOVDQA(X0, Mem{Base: R10}.Offset(0)) |
| XORQ(RCX, Mem{Base: R10}.Offset(0)) // 0(R10) = ·AVX_iv3 ^ (CX || 0) |
| |
| VMOVDQU(Mem{Base: AX}.Offset(0), X10) |
| VMOVDQU(Mem{Base: AX}.Offset(16), X11) |
| VMOVDQU(Mem{Base: AX}.Offset(32), X2) |
| VMOVDQU(Mem{Base: AX}.Offset(48), X3) |
| |
| MOVQ(Mem{Base: BX}.Offset(0), R8) |
| MOVQ(Mem{Base: BX}.Offset(8), R9) |
| |
| loop_AVX() |
| noinc_AVX() |
| } |
| |
| func loop_AVX() { |
| Label("loop") |
| ADDQ(Imm(128), R8) |
| CMPQ(R8, Imm(128)) |
| JGE(LabelRef("noinc")) |
| INCQ(R9) |
| } |
| |
| func noinc_AVX() { |
| Label("noinc") |
| VMOVQ_R8_X15() |
| VPINSRQ_1_R9_X15() |
| |
| AVX_iv0 := AVX_iv0_DATA() |
| AVX_iv1 := AVX_iv1_DATA() |
| AVX_iv2 := AVX_iv2_DATA() |
| VMOVDQA(X10, X0) |
| VMOVDQA(X11, X1) |
| VMOVDQU(AVX_iv0, X4) |
| VMOVDQU(AVX_iv1, X5) |
| VMOVDQU(AVX_iv2, X6) |
| |
| VPXOR(X15, X6, X6) |
| VMOVDQA(Mem{Base: R10}.Offset(0), X7) |
| |
| LOAD_MSG_AVX_0_2_4_6_1_3_5_7() |
| VMOVDQA(X12, Mem{Base: R10}.Offset(16)) |
| VMOVDQA(X13, Mem{Base: R10}.Offset(32)) |
| VMOVDQA(X14, Mem{Base: R10}.Offset(48)) |
| VMOVDQA(X15, Mem{Base: R10}.Offset(64)) |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX() |
| LOAD_MSG_AVX(8, 10, 12, 14, 9, 11, 13, 15) |
| VMOVDQA(X12, Mem{Base: R10}.Offset(80)) |
| VMOVDQA(X13, Mem{Base: R10}.Offset(96)) |
| VMOVDQA(X14, Mem{Base: R10}.Offset(112)) |
| VMOVDQA(X15, Mem{Base: R10}.Offset(128)) |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX_INV() |
| |
| LOAD_MSG_AVX(14, 4, 9, 13, 10, 8, 15, 6) |
| VMOVDQA(X12, Mem{Base: R10}.Offset(144)) |
| VMOVDQA(X13, Mem{Base: R10}.Offset(160)) |
| VMOVDQA(X14, Mem{Base: R10}.Offset(176)) |
| VMOVDQA(X15, Mem{Base: R10}.Offset(192)) |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX() |
| LOAD_MSG_AVX_1_0_11_5_12_2_7_3() |
| VMOVDQA(X12, Mem{Base: R10}.Offset(208)) |
| VMOVDQA(X13, Mem{Base: R10}.Offset(224)) |
| VMOVDQA(X14, Mem{Base: R10}.Offset(240)) |
| VMOVDQA(X15, Mem{Base: R10}.Offset(256)) |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX_INV() |
| |
| LOAD_MSG_AVX_11_12_5_15_8_0_2_13() |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX() |
| LOAD_MSG_AVX(10, 3, 7, 9, 14, 6, 1, 4) |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX_INV() |
| |
| LOAD_MSG_AVX(7, 3, 13, 11, 9, 1, 12, 14) |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX() |
| LOAD_MSG_AVX_2_5_4_15_6_10_0_8() |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX_INV() |
| |
| LOAD_MSG_AVX_9_5_2_10_0_7_4_15() |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX() |
| LOAD_MSG_AVX(14, 11, 6, 3, 1, 12, 8, 13) |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX_INV() |
| |
| LOAD_MSG_AVX_2_6_0_8_12_10_11_3() |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX() |
| LOAD_MSG_AVX(4, 7, 15, 1, 13, 5, 14, 9) |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX_INV() |
| |
| LOAD_MSG_AVX(12, 1, 14, 4, 5, 15, 13, 10) |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX() |
| LOAD_MSG_AVX_0_6_9_8_7_3_2_11() |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX_INV() |
| |
| LOAD_MSG_AVX(13, 7, 12, 3, 11, 14, 1, 9) |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX() |
| LOAD_MSG_AVX_5_15_8_2_0_4_6_10() |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX_INV() |
| |
| LOAD_MSG_AVX_6_14_11_0_15_9_3_8() |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX() |
| LOAD_MSG_AVX_12_13_1_10_2_7_4_5() |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX_INV() |
| |
| LOAD_MSG_AVX(10, 8, 7, 1, 2, 4, 6, 5) |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX() |
| LOAD_MSG_AVX_15_9_3_13_11_14_12_0() |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9) |
| SHUFFLE_AVX_INV() |
| |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(16), Mem{Base: R10}.Offset(32), Mem{Base: R10}.Offset(48), Mem{Base: R10}.Offset(64), X15, X8, X9) |
| SHUFFLE_AVX() |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(80), Mem{Base: R10}.Offset(96), Mem{Base: R10}.Offset(112), Mem{Base: R10}.Offset(128), X15, X8, X9) |
| SHUFFLE_AVX_INV() |
| |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(144), Mem{Base: R10}.Offset(160), Mem{Base: R10}.Offset(176), Mem{Base: R10}.Offset(192), X15, X8, X9) |
| SHUFFLE_AVX() |
| HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(208), Mem{Base: R10}.Offset(224), Mem{Base: R10}.Offset(240), Mem{Base: R10}.Offset(256), X15, X8, X9) |
| SHUFFLE_AVX_INV() |
| |
| VMOVDQU(Mem{Base: AX}.Offset(32), X14) |
| VMOVDQU(Mem{Base: AX}.Offset(48), X15) |
| VPXOR(X0, X10, X10) |
| VPXOR(X1, X11, X11) |
| VPXOR(X2, X14, X14) |
| VPXOR(X3, X15, X15) |
| VPXOR(X4, X10, X10) |
| VPXOR(X5, X11, X11) |
| VPXOR(X6, X14, X2) |
| VPXOR(X7, X15, X3) |
| VMOVDQU(X2, Mem{Base: AX}.Offset(32)) |
| VMOVDQU(X3, Mem{Base: AX}.Offset(48)) |
| |
| LEAQ(Mem{Base: SI}.Offset(128), RSI) |
| SUBQ(Imm(128), RDI) |
| JNE(LabelRef("loop")) |
| |
| VMOVDQU(X10, Mem{Base: AX}.Offset(0)) |
| VMOVDQU(X11, Mem{Base: AX}.Offset(16)) |
| |
| MOVQ(R8, Mem{Base: BX}.Offset(0)) |
| MOVQ(R9, Mem{Base: BX}.Offset(8)) |
| VZEROUPPER() |
| |
| RET() |
| } |
| |
| // ##~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~DATA SECTION~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~## |
| |
| var ( |
| AVX2_iv0_ptr, |
| AVX2_iv1_ptr, |
| AVX2_c40_ptr, |
| AVX2_c48_ptr, |
| |
| AVX_iv0_ptr, |
| AVX_iv1_ptr, |
| AVX_iv2_ptr, |
| AVX_iv3_ptr, |
| AVX_c40_ptr, |
| AVX_c48_ptr *Mem |
| ) |
| |
| func AVX2_iv0_DATA() Mem { |
| if AVX2_iv0_ptr != nil { |
| return *AVX2_iv0_ptr |
| } |
| AVX2_iv0 := GLOBL(ThatPeskyUnicodeDot+"AVX2_iv0", NOPTR|RODATA) |
| DATA(0x00, U64(0x6a09e667f3bcc908)) |
| DATA(0x08, U64(0xbb67ae8584caa73b)) |
| DATA(0x10, U64(0x3c6ef372fe94f82b)) |
| DATA(0x18, U64(0xa54ff53a5f1d36f1)) |
| return AVX2_iv0 |
| } |
| |
| func AVX2_iv1_DATA() Mem { |
| if AVX2_iv1_ptr != nil { |
| return *AVX2_iv1_ptr |
| } |
| AVX2_iv1 := GLOBL(ThatPeskyUnicodeDot+"AVX2_iv1", NOPTR|RODATA) |
| DATA(0x00, U64(0x510e527fade682d1)) |
| DATA(0x08, U64(0x9b05688c2b3e6c1f)) |
| DATA(0x10, U64(0x1f83d9abfb41bd6b)) |
| DATA(0x18, U64(0x5be0cd19137e2179)) |
| return AVX2_iv1 |
| } |
| |
| func AVX2_c40_DATA() Mem { |
| if AVX2_c40_ptr != nil { |
| return *AVX2_c40_ptr |
| } |
| AVX2_c40 := GLOBL(ThatPeskyUnicodeDot+"AVX2_c40", NOPTR|RODATA) |
| DATA(0x00, U64(0x0201000706050403)) |
| DATA(0x08, U64(0x0a09080f0e0d0c0b)) |
| DATA(0x10, U64(0x0201000706050403)) |
| DATA(0x18, U64(0x0a09080f0e0d0c0b)) |
| return AVX2_c40 |
| } |
| |
| func AVX2_c48_DATA() Mem { |
| if AVX2_c48_ptr != nil { |
| return *AVX2_c48_ptr |
| } |
| AVX2_c48 := GLOBL(ThatPeskyUnicodeDot+"AVX2_c48", NOPTR|RODATA) |
| DATA(0x00, U64(0x0100070605040302)) |
| DATA(0x08, U64(0x09080f0e0d0c0b0a)) |
| DATA(0x10, U64(0x0100070605040302)) |
| DATA(0x18, U64(0x09080f0e0d0c0b0a)) |
| return AVX2_c48 |
| } |
| |
| func AVX_iv0_DATA() Mem { |
| if AVX_iv0_ptr != nil { |
| return *AVX_iv0_ptr |
| } |
| AVX_iv0 := GLOBL(ThatPeskyUnicodeDot+"AVX_iv0", NOPTR|RODATA) |
| DATA(0x00, U64(0x6a09e667f3bcc908)) |
| DATA(0x08, U64(0xbb67ae8584caa73b)) |
| return AVX_iv0 |
| } |
| |
| func AVX_iv1_DATA() Mem { |
| if AVX_iv1_ptr != nil { |
| return *AVX_iv1_ptr |
| } |
| AVX_iv1 := GLOBL(ThatPeskyUnicodeDot+"AVX_iv1", NOPTR|RODATA) |
| DATA(0x00, U64(0x3c6ef372fe94f82b)) |
| DATA(0x08, U64(0xa54ff53a5f1d36f1)) |
| return AVX_iv1 |
| } |
| |
| func AVX_iv2_DATA() Mem { |
| if AVX_iv2_ptr != nil { |
| return *AVX_iv2_ptr |
| } |
| AVX_iv2 := GLOBL(ThatPeskyUnicodeDot+"AVX_iv2", NOPTR|RODATA) |
| DATA(0x00, U64(0x510e527fade682d1)) |
| DATA(0x08, U64(0x9b05688c2b3e6c1f)) |
| return AVX_iv2 |
| } |
| |
| func AVX_iv3_DATA() Mem { |
| if AVX_iv3_ptr != nil { |
| return *AVX_iv3_ptr |
| } |
| AVX_iv3 := GLOBL(ThatPeskyUnicodeDot+"AVX_iv3", NOPTR|RODATA) |
| DATA(0x00, U64(0x1f83d9abfb41bd6b)) |
| DATA(0x08, U64(0x5be0cd19137e2179)) |
| return AVX_iv3 |
| } |
| |
| func AVX_c40_DATA() Mem { |
| if AVX_c40_ptr != nil { |
| return *AVX_c40_ptr |
| } |
| AVX_c40 := GLOBL(ThatPeskyUnicodeDot+"AVX_c40", NOPTR|RODATA) |
| DATA(0x00, U64(0x0201000706050403)) |
| DATA(0x08, U64(0x0a09080f0e0d0c0b)) |
| return AVX_c40 |
| } |
| |
| func AVX_c48_DATA() Mem { |
| if AVX_c48_ptr != nil { |
| return *AVX_c48_ptr |
| } |
| AVX_c48 := GLOBL(ThatPeskyUnicodeDot+"AVX_c48", NOPTR|RODATA) |
| DATA(0x00, U64(0x0100070605040302)) |
| DATA(0x08, U64(0x09080f0e0d0c0b0a)) |
| return AVX_c48 |
| } |
