| // Copyright 2013 The Go Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style | |
| // license that can be found in the LICENSE file. | |
| // AVX2 version by Intel, same algorithm as code in Linux kernel: | |
| // https://github.com/torvalds/linux/blob/master/arch/x86/crypto/sha1_avx2_x86_64_asm.S | |
| // Authors: | |
| // Ilya Albrekht <ilya.albrekht@intel.com> | |
| // Maxim Locktyukhin <maxim.locktyukhin@intel.com> | |
| // Ronen Zohar <ronen.zohar@intel.com> | |
| // Chandramouli Narayanan <mouli@linux.intel.com> | |
| #include "textflag.h" | |
| // SHA-1 block routine. See sha1block.go for Go equivalent. | |
| // | |
| // There are 80 rounds of 4 types: | |
| // - rounds 0-15 are type 1 and load data (ROUND1 macro). | |
| // - rounds 16-19 are type 1 and do not load data (ROUND1x macro). | |
| // - rounds 20-39 are type 2 and do not load data (ROUND2 macro). | |
| // - rounds 40-59 are type 3 and do not load data (ROUND3 macro). | |
| // - rounds 60-79 are type 4 and do not load data (ROUND4 macro). | |
| // | |
| // Each round loads or shuffles the data, then computes a per-round | |
| // function of b, c, d, and then mixes the result into and rotates the | |
| // five registers a, b, c, d, e holding the intermediate results. | |
| // | |
| // The register rotation is implemented by rotating the arguments to | |
| // the round macros instead of by explicit move instructions. | |
| #define LOAD(index) \ | |
| MOVL (index*4)(SI), R10; \ | |
| BSWAPL R10; \ | |
| MOVL R10, (index*4)(SP) | |
| #define SHUFFLE(index) \ | |
| MOVL (((index)&0xf)*4)(SP), R10; \ | |
| XORL (((index-3)&0xf)*4)(SP), R10; \ | |
| XORL (((index-8)&0xf)*4)(SP), R10; \ | |
| XORL (((index-14)&0xf)*4)(SP), R10; \ | |
| ROLL $1, R10; \ | |
| MOVL R10, (((index)&0xf)*4)(SP) | |
| #define FUNC1(a, b, c, d, e) \ | |
| MOVL d, R9; \ | |
| XORL c, R9; \ | |
| ANDL b, R9; \ | |
| XORL d, R9 | |
| #define FUNC2(a, b, c, d, e) \ | |
| MOVL b, R9; \ | |
| XORL c, R9; \ | |
| XORL d, R9 | |
| #define FUNC3(a, b, c, d, e) \ | |
| MOVL b, R8; \ | |
| ORL c, R8; \ | |
| ANDL d, R8; \ | |
| MOVL b, R9; \ | |
| ANDL c, R9; \ | |
| ORL R8, R9 | |
| #define FUNC4 FUNC2 | |
| #define MIX(a, b, c, d, e, const) \ | |
| ROLL $30, b; \ | |
| ADDL R9, e; \ | |
| MOVL a, R8; \ | |
| ROLL $5, R8; \ | |
| LEAL const(e)(R10*1), e; \ | |
| ADDL R8, e | |
| #define ROUND1(a, b, c, d, e, index) \ | |
| LOAD(index); \ | |
| FUNC1(a, b, c, d, e); \ | |
| MIX(a, b, c, d, e, 0x5A827999) | |
| #define ROUND1x(a, b, c, d, e, index) \ | |
| SHUFFLE(index); \ | |
| FUNC1(a, b, c, d, e); \ | |
| MIX(a, b, c, d, e, 0x5A827999) | |
| #define ROUND2(a, b, c, d, e, index) \ | |
| SHUFFLE(index); \ | |
| FUNC2(a, b, c, d, e); \ | |
| MIX(a, b, c, d, e, 0x6ED9EBA1) | |
| #define ROUND3(a, b, c, d, e, index) \ | |
| SHUFFLE(index); \ | |
| FUNC3(a, b, c, d, e); \ | |
| MIX(a, b, c, d, e, 0x8F1BBCDC) | |
| #define ROUND4(a, b, c, d, e, index) \ | |
| SHUFFLE(index); \ | |
| FUNC4(a, b, c, d, e); \ | |
| MIX(a, b, c, d, e, 0xCA62C1D6) | |
| TEXT ·blockAMD64(SB),NOSPLIT,$64-32 | |
| MOVQ dig+0(FP), BP | |
| MOVQ p_base+8(FP), SI | |
| MOVQ p_len+16(FP), DX | |
| SHRQ $6, DX | |
| SHLQ $6, DX | |
| LEAQ (SI)(DX*1), DI | |
| MOVL (0*4)(BP), AX | |
| MOVL (1*4)(BP), BX | |
| MOVL (2*4)(BP), CX | |
| MOVL (3*4)(BP), DX | |
| MOVL (4*4)(BP), BP | |
| CMPQ SI, DI | |
| JEQ end | |
| loop: | |
| MOVL AX, R11 | |
| MOVL BX, R12 | |
| MOVL CX, R13 | |
| MOVL DX, R14 | |
| MOVL BP, R15 | |
| ROUND1(AX, BX, CX, DX, BP, 0) | |
| ROUND1(BP, AX, BX, CX, DX, 1) | |
| ROUND1(DX, BP, AX, BX, CX, 2) | |
| ROUND1(CX, DX, BP, AX, BX, 3) | |
| ROUND1(BX, CX, DX, BP, AX, 4) | |
| ROUND1(AX, BX, CX, DX, BP, 5) | |
| ROUND1(BP, AX, BX, CX, DX, 6) | |
| ROUND1(DX, BP, AX, BX, CX, 7) | |
| ROUND1(CX, DX, BP, AX, BX, 8) | |
| ROUND1(BX, CX, DX, BP, AX, 9) | |
| ROUND1(AX, BX, CX, DX, BP, 10) | |
| ROUND1(BP, AX, BX, CX, DX, 11) | |
| ROUND1(DX, BP, AX, BX, CX, 12) | |
| ROUND1(CX, DX, BP, AX, BX, 13) | |
| ROUND1(BX, CX, DX, BP, AX, 14) | |
| ROUND1(AX, BX, CX, DX, BP, 15) | |
| ROUND1x(BP, AX, BX, CX, DX, 16) | |
| ROUND1x(DX, BP, AX, BX, CX, 17) | |
| ROUND1x(CX, DX, BP, AX, BX, 18) | |
| ROUND1x(BX, CX, DX, BP, AX, 19) | |
| ROUND2(AX, BX, CX, DX, BP, 20) | |
| ROUND2(BP, AX, BX, CX, DX, 21) | |
| ROUND2(DX, BP, AX, BX, CX, 22) | |
| ROUND2(CX, DX, BP, AX, BX, 23) | |
| ROUND2(BX, CX, DX, BP, AX, 24) | |
| ROUND2(AX, BX, CX, DX, BP, 25) | |
| ROUND2(BP, AX, BX, CX, DX, 26) | |
| ROUND2(DX, BP, AX, BX, CX, 27) | |
| ROUND2(CX, DX, BP, AX, BX, 28) | |
| ROUND2(BX, CX, DX, BP, AX, 29) | |
| ROUND2(AX, BX, CX, DX, BP, 30) | |
| ROUND2(BP, AX, BX, CX, DX, 31) | |
| ROUND2(DX, BP, AX, BX, CX, 32) | |
| ROUND2(CX, DX, BP, AX, BX, 33) | |
| ROUND2(BX, CX, DX, BP, AX, 34) | |
| ROUND2(AX, BX, CX, DX, BP, 35) | |
| ROUND2(BP, AX, BX, CX, DX, 36) | |
| ROUND2(DX, BP, AX, BX, CX, 37) | |
| ROUND2(CX, DX, BP, AX, BX, 38) | |
| ROUND2(BX, CX, DX, BP, AX, 39) | |
| ROUND3(AX, BX, CX, DX, BP, 40) | |
| ROUND3(BP, AX, BX, CX, DX, 41) | |
| ROUND3(DX, BP, AX, BX, CX, 42) | |
| ROUND3(CX, DX, BP, AX, BX, 43) | |
| ROUND3(BX, CX, DX, BP, AX, 44) | |
| ROUND3(AX, BX, CX, DX, BP, 45) | |
| ROUND3(BP, AX, BX, CX, DX, 46) | |
| ROUND3(DX, BP, AX, BX, CX, 47) | |
| ROUND3(CX, DX, BP, AX, BX, 48) | |
| ROUND3(BX, CX, DX, BP, AX, 49) | |
| ROUND3(AX, BX, CX, DX, BP, 50) | |
| ROUND3(BP, AX, BX, CX, DX, 51) | |
| ROUND3(DX, BP, AX, BX, CX, 52) | |
| ROUND3(CX, DX, BP, AX, BX, 53) | |
| ROUND3(BX, CX, DX, BP, AX, 54) | |
| ROUND3(AX, BX, CX, DX, BP, 55) | |
| ROUND3(BP, AX, BX, CX, DX, 56) | |
| ROUND3(DX, BP, AX, BX, CX, 57) | |
| ROUND3(CX, DX, BP, AX, BX, 58) | |
| ROUND3(BX, CX, DX, BP, AX, 59) | |
| ROUND4(AX, BX, CX, DX, BP, 60) | |
| ROUND4(BP, AX, BX, CX, DX, 61) | |
| ROUND4(DX, BP, AX, BX, CX, 62) | |
| ROUND4(CX, DX, BP, AX, BX, 63) | |
| ROUND4(BX, CX, DX, BP, AX, 64) | |
| ROUND4(AX, BX, CX, DX, BP, 65) | |
| ROUND4(BP, AX, BX, CX, DX, 66) | |
| ROUND4(DX, BP, AX, BX, CX, 67) | |
| ROUND4(CX, DX, BP, AX, BX, 68) | |
| ROUND4(BX, CX, DX, BP, AX, 69) | |
| ROUND4(AX, BX, CX, DX, BP, 70) | |
| ROUND4(BP, AX, BX, CX, DX, 71) | |
| ROUND4(DX, BP, AX, BX, CX, 72) | |
| ROUND4(CX, DX, BP, AX, BX, 73) | |
| ROUND4(BX, CX, DX, BP, AX, 74) | |
| ROUND4(AX, BX, CX, DX, BP, 75) | |
| ROUND4(BP, AX, BX, CX, DX, 76) | |
| ROUND4(DX, BP, AX, BX, CX, 77) | |
| ROUND4(CX, DX, BP, AX, BX, 78) | |
| ROUND4(BX, CX, DX, BP, AX, 79) | |
| ADDL R11, AX | |
| ADDL R12, BX | |
| ADDL R13, CX | |
| ADDL R14, DX | |
| ADDL R15, BP | |
| ADDQ $64, SI | |
| CMPQ SI, DI | |
| JB loop | |
| end: | |
| MOVQ dig+0(FP), DI | |
| MOVL AX, (0*4)(DI) | |
| MOVL BX, (1*4)(DI) | |
| MOVL CX, (2*4)(DI) | |
| MOVL DX, (3*4)(DI) | |
| MOVL BP, (4*4)(DI) | |
| RET | |
| // This is the implementation using AVX2, BMI1 and BMI2. It is based on: | |
| // "SHA-1 implementation with Intel(R) AVX2 instruction set extensions" | |
| // From http://software.intel.com/en-us/articles | |
| // (look for improving-the-performance-of-the-secure-hash-algorithm-1) | |
| // This implementation is 2x unrolled, and interleaves vector instructions, | |
| // used to precompute W, with scalar computation of current round | |
| // for optimal scheduling. | |
| // Trivial helper macros. | |
| #define UPDATE_HASH(A,TB,C,D,E) \ | |
| ADDL (R9), A \ | |
| MOVL A, (R9) \ | |
| ADDL 4(R9), TB \ | |
| MOVL TB, 4(R9) \ | |
| ADDL 8(R9), C \ | |
| MOVL C, 8(R9) \ | |
| ADDL 12(R9), D \ | |
| MOVL D, 12(R9) \ | |
| ADDL 16(R9), E \ | |
| MOVL E, 16(R9) | |
| // Helper macros for PRECALC, which does precomputations | |
| #define PRECALC_0(OFFSET) \ | |
| VMOVDQU OFFSET(R10),X0 | |
| #define PRECALC_1(OFFSET) \ | |
| VINSERTI128 $1, OFFSET(R13), Y0, Y0 | |
| #define PRECALC_2(YREG) \ | |
| VPSHUFB Y10, Y0, YREG | |
| #define PRECALC_4(YREG,K_OFFSET) \ | |
| VPADDD K_OFFSET(R8), YREG, Y0 | |
| #define PRECALC_7(OFFSET) \ | |
| VMOVDQU Y0, (OFFSET*2)(R14) | |
| // Message scheduling pre-compute for rounds 0-15 | |
| // R13 is a pointer to even 64-byte block | |
| // R10 is a pointer to odd 64-byte block | |
| // R14 is a pointer to temp buffer | |
| // X0 is used as temp register | |
| // YREG is clobbered as part of computation | |
| // OFFSET chooses 16 byte chunk within a block | |
| // R8 is a pointer to constants block | |
| // K_OFFSET chooses K constants relevant to this round | |
| // X10 holds swap mask | |
| #define PRECALC_00_15(OFFSET,YREG) \ | |
| PRECALC_0(OFFSET) \ | |
| PRECALC_1(OFFSET) \ | |
| PRECALC_2(YREG) \ | |
| PRECALC_4(YREG,0x0) \ | |
| PRECALC_7(OFFSET) | |
| // Helper macros for PRECALC_16_31 | |
| #define PRECALC_16(REG_SUB_16,REG_SUB_12,REG_SUB_4,REG) \ | |
| VPALIGNR $8, REG_SUB_16, REG_SUB_12, REG \ // w[i-14] | |
| VPSRLDQ $4, REG_SUB_4, Y0 // w[i-3] | |
| #define PRECALC_17(REG_SUB_16,REG_SUB_8,REG) \ | |
| VPXOR REG_SUB_8, REG, REG \ | |
| VPXOR REG_SUB_16, Y0, Y0 | |
| #define PRECALC_18(REG) \ | |
| VPXOR Y0, REG, REG \ | |
| VPSLLDQ $12, REG, Y9 | |
| #define PRECALC_19(REG) \ | |
| VPSLLD $1, REG, Y0 \ | |
| VPSRLD $31, REG, REG | |
| #define PRECALC_20(REG) \ | |
| VPOR REG, Y0, Y0 \ | |
| VPSLLD $2, Y9, REG | |
| #define PRECALC_21(REG) \ | |
| VPSRLD $30, Y9, Y9 \ | |
| VPXOR REG, Y0, Y0 | |
| #define PRECALC_23(REG,K_OFFSET,OFFSET) \ | |
| VPXOR Y9, Y0, REG \ | |
| VPADDD K_OFFSET(R8), REG, Y0 \ | |
| VMOVDQU Y0, (OFFSET)(R14) | |
| // Message scheduling pre-compute for rounds 16-31 | |
| // calculating last 32 w[i] values in 8 XMM registers | |
| // pre-calculate K+w[i] values and store to mem | |
| // for later load by ALU add instruction. | |
| // "brute force" vectorization for rounds 16-31 only | |
| // due to w[i]->w[i-3] dependency. | |
| // clobbers 5 input ymm registers REG_SUB* | |
| // uses X0 and X9 as temp registers | |
| // As always, R8 is a pointer to constants block | |
| // and R14 is a pointer to temp buffer | |
| #define PRECALC_16_31(REG,REG_SUB_4,REG_SUB_8,REG_SUB_12,REG_SUB_16,K_OFFSET,OFFSET) \ | |
| PRECALC_16(REG_SUB_16,REG_SUB_12,REG_SUB_4,REG) \ | |
| PRECALC_17(REG_SUB_16,REG_SUB_8,REG) \ | |
| PRECALC_18(REG) \ | |
| PRECALC_19(REG) \ | |
| PRECALC_20(REG) \ | |
| PRECALC_21(REG) \ | |
| PRECALC_23(REG,K_OFFSET,OFFSET) | |
| // Helper macros for PRECALC_32_79 | |
| #define PRECALC_32(REG_SUB_8,REG_SUB_4) \ | |
| VPALIGNR $8, REG_SUB_8, REG_SUB_4, Y0 | |
| #define PRECALC_33(REG_SUB_28,REG) \ | |
| VPXOR REG_SUB_28, REG, REG | |
| #define PRECALC_34(REG_SUB_16) \ | |
| VPXOR REG_SUB_16, Y0, Y0 | |
| #define PRECALC_35(REG) \ | |
| VPXOR Y0, REG, REG | |
| #define PRECALC_36(REG) \ | |
| VPSLLD $2, REG, Y0 | |
| #define PRECALC_37(REG) \ | |
| VPSRLD $30, REG, REG \ | |
| VPOR REG, Y0, REG | |
| #define PRECALC_39(REG,K_OFFSET,OFFSET) \ | |
| VPADDD K_OFFSET(R8), REG, Y0 \ | |
| VMOVDQU Y0, (OFFSET)(R14) | |
| // Message scheduling pre-compute for rounds 32-79 | |
| // In SHA-1 specification we have: | |
| // w[i] = (w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16]) rol 1 | |
| // Which is the same as: | |
| // w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2 | |
| // This allows for more efficient vectorization, | |
| // since w[i]->w[i-3] dependency is broken | |
| #define PRECALC_32_79(REG,REG_SUB_4,REG_SUB_8,REG_SUB_16,REG_SUB_28,K_OFFSET,OFFSET) \ | |
| PRECALC_32(REG_SUB_8,REG_SUB_4) \ | |
| PRECALC_33(REG_SUB_28,REG) \ | |
| PRECALC_34(REG_SUB_16) \ | |
| PRECALC_35(REG) \ | |
| PRECALC_36(REG) \ | |
| PRECALC_37(REG) \ | |
| PRECALC_39(REG,K_OFFSET,OFFSET) | |
| #define PRECALC \ | |
| PRECALC_00_15(0,Y15) \ | |
| PRECALC_00_15(0x10,Y14) \ | |
| PRECALC_00_15(0x20,Y13) \ | |
| PRECALC_00_15(0x30,Y12) \ | |
| PRECALC_16_31(Y8,Y12,Y13,Y14,Y15,0,0x80) \ | |
| PRECALC_16_31(Y7,Y8,Y12,Y13,Y14,0x20,0xa0) \ | |
| PRECALC_16_31(Y5,Y7,Y8,Y12,Y13,0x20,0xc0) \ | |
| PRECALC_16_31(Y3,Y5,Y7,Y8,Y12,0x20,0xe0) \ | |
| PRECALC_32_79(Y15,Y3,Y5,Y8,Y14,0x20,0x100) \ | |
| PRECALC_32_79(Y14,Y15,Y3,Y7,Y13,0x20,0x120) \ | |
| PRECALC_32_79(Y13,Y14,Y15,Y5,Y12,0x40,0x140) \ | |
| PRECALC_32_79(Y12,Y13,Y14,Y3,Y8,0x40,0x160) \ | |
| PRECALC_32_79(Y8,Y12,Y13,Y15,Y7,0x40,0x180) \ | |
| PRECALC_32_79(Y7,Y8,Y12,Y14,Y5,0x40,0x1a0) \ | |
| PRECALC_32_79(Y5,Y7,Y8,Y13,Y3,0x40,0x1c0) \ | |
| PRECALC_32_79(Y3,Y5,Y7,Y12,Y15,0x60,0x1e0) \ | |
| PRECALC_32_79(Y15,Y3,Y5,Y8,Y14,0x60,0x200) \ | |
| PRECALC_32_79(Y14,Y15,Y3,Y7,Y13,0x60,0x220) \ | |
| PRECALC_32_79(Y13,Y14,Y15,Y5,Y12,0x60,0x240) \ | |
| PRECALC_32_79(Y12,Y13,Y14,Y3,Y8,0x60,0x260) | |
| // Macros calculating individual rounds have general forn | |
| // CALC_ROUND_PRE + PRECALC_ROUND + CALC_ROUND_POST | |
| // CALC_ROUND_{PRE,POST} macros follow | |
| #define CALC_F1_PRE(OFFSET,REG_A,REG_B,REG_C,REG_E) \ | |
| ADDL OFFSET(R15),REG_E \ | |
| ANDNL REG_C,REG_A,BP \ | |
| LEAL (REG_E)(REG_B*1), REG_E \ // Add F from the previous round | |
| RORXL $0x1b, REG_A, R12 \ | |
| RORXL $2, REG_A, REG_B // for next round | |
| // Calculate F for the next round | |
| #define CALC_F1_POST(REG_A,REG_B,REG_E) \ | |
| ANDL REG_B,REG_A \ // b&c | |
| XORL BP, REG_A \ // F1 = (b&c) ^ (~b&d) | |
| LEAL (REG_E)(R12*1), REG_E // E += A >>> 5 | |
| // Registers are cycleickly rotated DX -> AX -> DI -> SI -> BX -> CX | |
| #define CALC_0 \ | |
| MOVL SI, BX \ // Precalculating first round | |
| RORXL $2, SI, SI \ | |
| ANDNL AX, BX, BP \ | |
| ANDL DI, BX \ | |
| XORL BP, BX \ | |
| CALC_F1_PRE(0x0,CX,BX,DI,DX) \ | |
| PRECALC_0(0x80) \ | |
| CALC_F1_POST(CX,SI,DX) | |
| #define CALC_1 \ | |
| CALC_F1_PRE(0x4,DX,CX,SI,AX) \ | |
| PRECALC_1(0x80) \ | |
| CALC_F1_POST(DX,BX,AX) | |
| #define CALC_2 \ | |
| CALC_F1_PRE(0x8,AX,DX,BX,DI) \ | |
| PRECALC_2(Y15) \ | |
| CALC_F1_POST(AX,CX,DI) | |
| #define CALC_3 \ | |
| CALC_F1_PRE(0xc,DI,AX,CX,SI) \ | |
| CALC_F1_POST(DI,DX,SI) | |
| #define CALC_4 \ | |
| CALC_F1_PRE(0x20,SI,DI,DX,BX) \ | |
| PRECALC_4(Y15,0x0) \ | |
| CALC_F1_POST(SI,AX,BX) | |
| #define CALC_5 \ | |
| CALC_F1_PRE(0x24,BX,SI,AX,CX) \ | |
| CALC_F1_POST(BX,DI,CX) | |
| #define CALC_6 \ | |
| CALC_F1_PRE(0x28,CX,BX,DI,DX) \ | |
| CALC_F1_POST(CX,SI,DX) | |
| #define CALC_7 \ | |
| CALC_F1_PRE(0x2c,DX,CX,SI,AX) \ | |
| PRECALC_7(0x0) \ | |
| CALC_F1_POST(DX,BX,AX) | |
| #define CALC_8 \ | |
| CALC_F1_PRE(0x40,AX,DX,BX,DI) \ | |
| PRECALC_0(0x90) \ | |
| CALC_F1_POST(AX,CX,DI) | |
| #define CALC_9 \ | |
| CALC_F1_PRE(0x44,DI,AX,CX,SI) \ | |
| PRECALC_1(0x90) \ | |
| CALC_F1_POST(DI,DX,SI) | |
| #define CALC_10 \ | |
| CALC_F1_PRE(0x48,SI,DI,DX,BX) \ | |
| PRECALC_2(Y14) \ | |
| CALC_F1_POST(SI,AX,BX) | |
| #define CALC_11 \ | |
| CALC_F1_PRE(0x4c,BX,SI,AX,CX) \ | |
| CALC_F1_POST(BX,DI,CX) | |
| #define CALC_12 \ | |
| CALC_F1_PRE(0x60,CX,BX,DI,DX) \ | |
| PRECALC_4(Y14,0x0) \ | |
| CALC_F1_POST(CX,SI,DX) | |
| #define CALC_13 \ | |
| CALC_F1_PRE(0x64,DX,CX,SI,AX) \ | |
| CALC_F1_POST(DX,BX,AX) | |
| #define CALC_14 \ | |
| CALC_F1_PRE(0x68,AX,DX,BX,DI) \ | |
| CALC_F1_POST(AX,CX,DI) | |
| #define CALC_15 \ | |
| CALC_F1_PRE(0x6c,DI,AX,CX,SI) \ | |
| PRECALC_7(0x10) \ | |
| CALC_F1_POST(DI,DX,SI) | |
| #define CALC_16 \ | |
| CALC_F1_PRE(0x80,SI,DI,DX,BX) \ | |
| PRECALC_0(0xa0) \ | |
| CALC_F1_POST(SI,AX,BX) | |
| #define CALC_17 \ | |
| CALC_F1_PRE(0x84,BX,SI,AX,CX) \ | |
| PRECALC_1(0xa0) \ | |
| CALC_F1_POST(BX,DI,CX) | |
| #define CALC_18 \ | |
| CALC_F1_PRE(0x88,CX,BX,DI,DX) \ | |
| PRECALC_2(Y13) \ | |
| CALC_F1_POST(CX,SI,DX) | |
| #define CALC_F2_PRE(OFFSET,REG_A,REG_B,REG_E) \ | |
| ADDL OFFSET(R15),REG_E \ | |
| LEAL (REG_E)(REG_B*1), REG_E \ // Add F from the previous round | |
| RORXL $0x1b, REG_A, R12 \ | |
| RORXL $2, REG_A, REG_B // for next round | |
| #define CALC_F2_POST(REG_A,REG_B,REG_C,REG_E) \ | |
| XORL REG_B, REG_A \ | |
| ADDL R12, REG_E \ | |
| XORL REG_C, REG_A | |
| #define CALC_19 \ | |
| CALC_F2_PRE(0x8c,DX,CX,AX) \ | |
| CALC_F2_POST(DX,BX,SI,AX) | |
| #define CALC_20 \ | |
| CALC_F2_PRE(0xa0,AX,DX,DI) \ | |
| PRECALC_4(Y13,0x0) \ | |
| CALC_F2_POST(AX,CX,BX,DI) | |
| #define CALC_21 \ | |
| CALC_F2_PRE(0xa4,DI,AX,SI) \ | |
| CALC_F2_POST(DI,DX,CX,SI) | |
| #define CALC_22 \ | |
| CALC_F2_PRE(0xa8,SI,DI,BX) \ | |
| CALC_F2_POST(SI,AX,DX,BX) | |
| #define CALC_23 \ | |
| CALC_F2_PRE(0xac,BX,SI,CX) \ | |
| PRECALC_7(0x20) \ | |
| CALC_F2_POST(BX,DI,AX,CX) | |
| #define CALC_24 \ | |
| CALC_F2_PRE(0xc0,CX,BX,DX) \ | |
| PRECALC_0(0xb0) \ | |
| CALC_F2_POST(CX,SI,DI,DX) | |
| #define CALC_25 \ | |
| CALC_F2_PRE(0xc4,DX,CX,AX) \ | |
| PRECALC_1(0xb0) \ | |
| CALC_F2_POST(DX,BX,SI,AX) | |
| #define CALC_26 \ | |
| CALC_F2_PRE(0xc8,AX,DX,DI) \ | |
| PRECALC_2(Y12) \ | |
| CALC_F2_POST(AX,CX,BX,DI) | |
| #define CALC_27 \ | |
| CALC_F2_PRE(0xcc,DI,AX,SI) \ | |
| CALC_F2_POST(DI,DX,CX,SI) | |
| #define CALC_28 \ | |
| CALC_F2_PRE(0xe0,SI,DI,BX) \ | |
| PRECALC_4(Y12,0x0) \ | |
| CALC_F2_POST(SI,AX,DX,BX) | |
| #define CALC_29 \ | |
| CALC_F2_PRE(0xe4,BX,SI,CX) \ | |
| CALC_F2_POST(BX,DI,AX,CX) | |
| #define CALC_30 \ | |
| CALC_F2_PRE(0xe8,CX,BX,DX) \ | |
| CALC_F2_POST(CX,SI,DI,DX) | |
| #define CALC_31 \ | |
| CALC_F2_PRE(0xec,DX,CX,AX) \ | |
| PRECALC_7(0x30) \ | |
| CALC_F2_POST(DX,BX,SI,AX) | |
| #define CALC_32 \ | |
| CALC_F2_PRE(0x100,AX,DX,DI) \ | |
| PRECALC_16(Y15,Y14,Y12,Y8) \ | |
| CALC_F2_POST(AX,CX,BX,DI) | |
| #define CALC_33 \ | |
| CALC_F2_PRE(0x104,DI,AX,SI) \ | |
| PRECALC_17(Y15,Y13,Y8) \ | |
| CALC_F2_POST(DI,DX,CX,SI) | |
| #define CALC_34 \ | |
| CALC_F2_PRE(0x108,SI,DI,BX) \ | |
| PRECALC_18(Y8) \ | |
| CALC_F2_POST(SI,AX,DX,BX) | |
| #define CALC_35 \ | |
| CALC_F2_PRE(0x10c,BX,SI,CX) \ | |
| PRECALC_19(Y8) \ | |
| CALC_F2_POST(BX,DI,AX,CX) | |
| #define CALC_36 \ | |
| CALC_F2_PRE(0x120,CX,BX,DX) \ | |
| PRECALC_20(Y8) \ | |
| CALC_F2_POST(CX,SI,DI,DX) | |
| #define CALC_37 \ | |
| CALC_F2_PRE(0x124,DX,CX,AX) \ | |
| PRECALC_21(Y8) \ | |
| CALC_F2_POST(DX,BX,SI,AX) | |
| #define CALC_38 \ | |
| CALC_F2_PRE(0x128,AX,DX,DI) \ | |
| CALC_F2_POST(AX,CX,BX,DI) | |
| #define CALC_F3_PRE(OFFSET,REG_E) \ | |
| ADDL OFFSET(R15),REG_E | |
| #define CALC_F3_POST(REG_A,REG_B,REG_C,REG_E,REG_TB) \ | |
| LEAL (REG_E)(REG_TB*1), REG_E \ // Add F from the previous round | |
| MOVL REG_B, BP \ | |
| ORL REG_A, BP \ | |
| RORXL $0x1b, REG_A, R12 \ | |
| RORXL $2, REG_A, REG_TB \ | |
| ANDL REG_C, BP \ // Calculate F for the next round | |
| ANDL REG_B, REG_A \ | |
| ORL BP, REG_A \ | |
| ADDL R12, REG_E | |
| #define CALC_39 \ | |
| CALC_F3_PRE(0x12c,SI) \ | |
| PRECALC_23(Y8,0x0,0x80) \ | |
| CALC_F3_POST(DI,DX,CX,SI,AX) | |
| #define CALC_40 \ | |
| CALC_F3_PRE(0x140,BX) \ | |
| PRECALC_16(Y14,Y13,Y8,Y7) \ | |
| CALC_F3_POST(SI,AX,DX,BX,DI) | |
| #define CALC_41 \ | |
| CALC_F3_PRE(0x144,CX) \ | |
| PRECALC_17(Y14,Y12,Y7) \ | |
| CALC_F3_POST(BX,DI,AX,CX,SI) | |
| #define CALC_42 \ | |
| CALC_F3_PRE(0x148,DX) \ | |
| PRECALC_18(Y7) \ | |
| CALC_F3_POST(CX,SI,DI,DX,BX) | |
| #define CALC_43 \ | |
| CALC_F3_PRE(0x14c,AX) \ | |
| PRECALC_19(Y7) \ | |
| CALC_F3_POST(DX,BX,SI,AX,CX) | |
| #define CALC_44 \ | |
| CALC_F3_PRE(0x160,DI) \ | |
| PRECALC_20(Y7) \ | |
| CALC_F3_POST(AX,CX,BX,DI,DX) | |
| #define CALC_45 \ | |
| CALC_F3_PRE(0x164,SI) \ | |
| PRECALC_21(Y7) \ | |
| CALC_F3_POST(DI,DX,CX,SI,AX) | |
| #define CALC_46 \ | |
| CALC_F3_PRE(0x168,BX) \ | |
| CALC_F3_POST(SI,AX,DX,BX,DI) | |
| #define CALC_47 \ | |
| CALC_F3_PRE(0x16c,CX) \ | |
| VPXOR Y9, Y0, Y7 \ | |
| VPADDD 0x20(R8), Y7, Y0 \ | |
| VMOVDQU Y0, 0xa0(R14) \ | |
| CALC_F3_POST(BX,DI,AX,CX,SI) | |
| #define CALC_48 \ | |
| CALC_F3_PRE(0x180,DX) \ | |
| PRECALC_16(Y13,Y12,Y7,Y5) \ | |
| CALC_F3_POST(CX,SI,DI,DX,BX) | |
| #define CALC_49 \ | |
| CALC_F3_PRE(0x184,AX) \ | |
| PRECALC_17(Y13,Y8,Y5) \ | |
| CALC_F3_POST(DX,BX,SI,AX,CX) | |
| #define CALC_50 \ | |
| CALC_F3_PRE(0x188,DI) \ | |
| PRECALC_18(Y5) \ | |
| CALC_F3_POST(AX,CX,BX,DI,DX) | |
| #define CALC_51 \ | |
| CALC_F3_PRE(0x18c,SI) \ | |
| PRECALC_19(Y5) \ | |
| CALC_F3_POST(DI,DX,CX,SI,AX) | |
| #define CALC_52 \ | |
| CALC_F3_PRE(0x1a0,BX) \ | |
| PRECALC_20(Y5) \ | |
| CALC_F3_POST(SI,AX,DX,BX,DI) | |
| #define CALC_53 \ | |
| CALC_F3_PRE(0x1a4,CX) \ | |
| PRECALC_21(Y5) \ | |
| CALC_F3_POST(BX,DI,AX,CX,SI) | |
| #define CALC_54 \ | |
| CALC_F3_PRE(0x1a8,DX) \ | |
| CALC_F3_POST(CX,SI,DI,DX,BX) | |
| #define CALC_55 \ | |
| CALC_F3_PRE(0x1ac,AX) \ | |
| PRECALC_23(Y5,0x20,0xc0) \ | |
| CALC_F3_POST(DX,BX,SI,AX,CX) | |
| #define CALC_56 \ | |
| CALC_F3_PRE(0x1c0,DI) \ | |
| PRECALC_16(Y12,Y8,Y5,Y3) \ | |
| CALC_F3_POST(AX,CX,BX,DI,DX) | |
| #define CALC_57 \ | |
| CALC_F3_PRE(0x1c4,SI) \ | |
| PRECALC_17(Y12,Y7,Y3) \ | |
| CALC_F3_POST(DI,DX,CX,SI,AX) | |
| #define CALC_58 \ | |
| CALC_F3_PRE(0x1c8,BX) \ | |
| PRECALC_18(Y3) \ | |
| CALC_F3_POST(SI,AX,DX,BX,DI) | |
| #define CALC_59 \ | |
| CALC_F2_PRE(0x1cc,BX,SI,CX) \ | |
| PRECALC_19(Y3) \ | |
| CALC_F2_POST(BX,DI,AX,CX) | |
| #define CALC_60 \ | |
| CALC_F2_PRE(0x1e0,CX,BX,DX) \ | |
| PRECALC_20(Y3) \ | |
| CALC_F2_POST(CX,SI,DI,DX) | |
| #define CALC_61 \ | |
| CALC_F2_PRE(0x1e4,DX,CX,AX) \ | |
| PRECALC_21(Y3) \ | |
| CALC_F2_POST(DX,BX,SI,AX) | |
| #define CALC_62 \ | |
| CALC_F2_PRE(0x1e8,AX,DX,DI) \ | |
| CALC_F2_POST(AX,CX,BX,DI) | |
| #define CALC_63 \ | |
| CALC_F2_PRE(0x1ec,DI,AX,SI) \ | |
| PRECALC_23(Y3,0x20,0xe0) \ | |
| CALC_F2_POST(DI,DX,CX,SI) | |
| #define CALC_64 \ | |
| CALC_F2_PRE(0x200,SI,DI,BX) \ | |
| PRECALC_32(Y5,Y3) \ | |
| CALC_F2_POST(SI,AX,DX,BX) | |
| #define CALC_65 \ | |
| CALC_F2_PRE(0x204,BX,SI,CX) \ | |
| PRECALC_33(Y14,Y15) \ | |
| CALC_F2_POST(BX,DI,AX,CX) | |
| #define CALC_66 \ | |
| CALC_F2_PRE(0x208,CX,BX,DX) \ | |
| PRECALC_34(Y8) \ | |
| CALC_F2_POST(CX,SI,DI,DX) | |
| #define CALC_67 \ | |
| CALC_F2_PRE(0x20c,DX,CX,AX) \ | |
| PRECALC_35(Y15) \ | |
| CALC_F2_POST(DX,BX,SI,AX) | |
| #define CALC_68 \ | |
| CALC_F2_PRE(0x220,AX,DX,DI) \ | |
| PRECALC_36(Y15) \ | |
| CALC_F2_POST(AX,CX,BX,DI) | |
| #define CALC_69 \ | |
| CALC_F2_PRE(0x224,DI,AX,SI) \ | |
| PRECALC_37(Y15) \ | |
| CALC_F2_POST(DI,DX,CX,SI) | |
| #define CALC_70 \ | |
| CALC_F2_PRE(0x228,SI,DI,BX) \ | |
| CALC_F2_POST(SI,AX,DX,BX) | |
| #define CALC_71 \ | |
| CALC_F2_PRE(0x22c,BX,SI,CX) \ | |
| PRECALC_39(Y15,0x20,0x100) \ | |
| CALC_F2_POST(BX,DI,AX,CX) | |
| #define CALC_72 \ | |
| CALC_F2_PRE(0x240,CX,BX,DX) \ | |
| PRECALC_32(Y3,Y15) \ | |
| CALC_F2_POST(CX,SI,DI,DX) | |
| #define CALC_73 \ | |
| CALC_F2_PRE(0x244,DX,CX,AX) \ | |
| PRECALC_33(Y13,Y14) \ | |
| CALC_F2_POST(DX,BX,SI,AX) | |
| #define CALC_74 \ | |
| CALC_F2_PRE(0x248,AX,DX,DI) \ | |
| PRECALC_34(Y7) \ | |
| CALC_F2_POST(AX,CX,BX,DI) | |
| #define CALC_75 \ | |
| CALC_F2_PRE(0x24c,DI,AX,SI) \ | |
| PRECALC_35(Y14) \ | |
| CALC_F2_POST(DI,DX,CX,SI) | |
| #define CALC_76 \ | |
| CALC_F2_PRE(0x260,SI,DI,BX) \ | |
| PRECALC_36(Y14) \ | |
| CALC_F2_POST(SI,AX,DX,BX) | |
| #define CALC_77 \ | |
| CALC_F2_PRE(0x264,BX,SI,CX) \ | |
| PRECALC_37(Y14) \ | |
| CALC_F2_POST(BX,DI,AX,CX) | |
| #define CALC_78 \ | |
| CALC_F2_PRE(0x268,CX,BX,DX) \ | |
| CALC_F2_POST(CX,SI,DI,DX) | |
| #define CALC_79 \ | |
| ADDL 0x26c(R15), AX \ | |
| LEAL (AX)(CX*1), AX \ | |
| RORXL $0x1b, DX, R12 \ | |
| PRECALC_39(Y14,0x20,0x120) \ | |
| ADDL R12, AX | |
| // Similar to CALC_0 | |
| #define CALC_80 \ | |
| MOVL CX, DX \ | |
| RORXL $2, CX, CX \ | |
| ANDNL SI, DX, BP \ | |
| ANDL BX, DX \ | |
| XORL BP, DX \ | |
| CALC_F1_PRE(0x10,AX,DX,BX,DI) \ | |
| PRECALC_32(Y15,Y14) \ | |
| CALC_F1_POST(AX,CX,DI) | |
| #define CALC_81 \ | |
| CALC_F1_PRE(0x14,DI,AX,CX,SI) \ | |
| PRECALC_33(Y12,Y13) \ | |
| CALC_F1_POST(DI,DX,SI) | |
| #define CALC_82 \ | |
| CALC_F1_PRE(0x18,SI,DI,DX,BX) \ | |
| PRECALC_34(Y5) \ | |
| CALC_F1_POST(SI,AX,BX) | |
| #define CALC_83 \ | |
| CALC_F1_PRE(0x1c,BX,SI,AX,CX) \ | |
| PRECALC_35(Y13) \ | |
| CALC_F1_POST(BX,DI,CX) | |
| #define CALC_84 \ | |
| CALC_F1_PRE(0x30,CX,BX,DI,DX) \ | |
| PRECALC_36(Y13) \ | |
| CALC_F1_POST(CX,SI,DX) | |
| #define CALC_85 \ | |
| CALC_F1_PRE(0x34,DX,CX,SI,AX) \ | |
| PRECALC_37(Y13) \ | |
| CALC_F1_POST(DX,BX,AX) | |
| #define CALC_86 \ | |
| CALC_F1_PRE(0x38,AX,DX,BX,DI) \ | |
| CALC_F1_POST(AX,CX,DI) | |
| #define CALC_87 \ | |
| CALC_F1_PRE(0x3c,DI,AX,CX,SI) \ | |
| PRECALC_39(Y13,0x40,0x140) \ | |
| CALC_F1_POST(DI,DX,SI) | |
| #define CALC_88 \ | |
| CALC_F1_PRE(0x50,SI,DI,DX,BX) \ | |
| PRECALC_32(Y14,Y13) \ | |
| CALC_F1_POST(SI,AX,BX) | |
| #define CALC_89 \ | |
| CALC_F1_PRE(0x54,BX,SI,AX,CX) \ | |
| PRECALC_33(Y8,Y12) \ | |
| CALC_F1_POST(BX,DI,CX) | |
| #define CALC_90 \ | |
| CALC_F1_PRE(0x58,CX,BX,DI,DX) \ | |
| PRECALC_34(Y3) \ | |
| CALC_F1_POST(CX,SI,DX) | |
| #define CALC_91 \ | |
| CALC_F1_PRE(0x5c,DX,CX,SI,AX) \ | |
| PRECALC_35(Y12) \ | |
| CALC_F1_POST(DX,BX,AX) | |
| #define CALC_92 \ | |
| CALC_F1_PRE(0x70,AX,DX,BX,DI) \ | |
| PRECALC_36(Y12) \ | |
| CALC_F1_POST(AX,CX,DI) | |
| #define CALC_93 \ | |
| CALC_F1_PRE(0x74,DI,AX,CX,SI) \ | |
| PRECALC_37(Y12) \ | |
| CALC_F1_POST(DI,DX,SI) | |
| #define CALC_94 \ | |
| CALC_F1_PRE(0x78,SI,DI,DX,BX) \ | |
| CALC_F1_POST(SI,AX,BX) | |
| #define CALC_95 \ | |
| CALC_F1_PRE(0x7c,BX,SI,AX,CX) \ | |
| PRECALC_39(Y12,0x40,0x160) \ | |
| CALC_F1_POST(BX,DI,CX) | |
| #define CALC_96 \ | |
| CALC_F1_PRE(0x90,CX,BX,DI,DX) \ | |
| PRECALC_32(Y13,Y12) \ | |
| CALC_F1_POST(CX,SI,DX) | |
| #define CALC_97 \ | |
| CALC_F1_PRE(0x94,DX,CX,SI,AX) \ | |
| PRECALC_33(Y7,Y8) \ | |
| CALC_F1_POST(DX,BX,AX) | |
| #define CALC_98 \ | |
| CALC_F1_PRE(0x98,AX,DX,BX,DI) \ | |
| PRECALC_34(Y15) \ | |
| CALC_F1_POST(AX,CX,DI) | |
| #define CALC_99 \ | |
| CALC_F2_PRE(0x9c,DI,AX,SI) \ | |
| PRECALC_35(Y8) \ | |
| CALC_F2_POST(DI,DX,CX,SI) | |
| #define CALC_100 \ | |
| CALC_F2_PRE(0xb0,SI,DI,BX) \ | |
| PRECALC_36(Y8) \ | |
| CALC_F2_POST(SI,AX,DX,BX) | |
| #define CALC_101 \ | |
| CALC_F2_PRE(0xb4,BX,SI,CX) \ | |
| PRECALC_37(Y8) \ | |
| CALC_F2_POST(BX,DI,AX,CX) | |
| #define CALC_102 \ | |
| CALC_F2_PRE(0xb8,CX,BX,DX) \ | |
| CALC_F2_POST(CX,SI,DI,DX) | |
| #define CALC_103 \ | |
| CALC_F2_PRE(0xbc,DX,CX,AX) \ | |
| PRECALC_39(Y8,0x40,0x180) \ | |
| CALC_F2_POST(DX,BX,SI,AX) | |
| #define CALC_104 \ | |
| CALC_F2_PRE(0xd0,AX,DX,DI) \ | |
| PRECALC_32(Y12,Y8) \ | |
| CALC_F2_POST(AX,CX,BX,DI) | |
| #define CALC_105 \ | |
| CALC_F2_PRE(0xd4,DI,AX,SI) \ | |
| PRECALC_33(Y5,Y7) \ | |
| CALC_F2_POST(DI,DX,CX,SI) | |
| #define CALC_106 \ | |
| CALC_F2_PRE(0xd8,SI,DI,BX) \ | |
| PRECALC_34(Y14) \ | |
| CALC_F2_POST(SI,AX,DX,BX) | |
| #define CALC_107 \ | |
| CALC_F2_PRE(0xdc,BX,SI,CX) \ | |
| PRECALC_35(Y7) \ | |
| CALC_F2_POST(BX,DI,AX,CX) | |
| #define CALC_108 \ | |
| CALC_F2_PRE(0xf0,CX,BX,DX) \ | |
| PRECALC_36(Y7) \ | |
| CALC_F2_POST(CX,SI,DI,DX) | |
| #define CALC_109 \ | |
| CALC_F2_PRE(0xf4,DX,CX,AX) \ | |
| PRECALC_37(Y7) \ | |
| CALC_F2_POST(DX,BX,SI,AX) | |
| #define CALC_110 \ | |
| CALC_F2_PRE(0xf8,AX,DX,DI) \ | |
| CALC_F2_POST(AX,CX,BX,DI) | |
| #define CALC_111 \ | |
| CALC_F2_PRE(0xfc,DI,AX,SI) \ | |
| PRECALC_39(Y7,0x40,0x1a0) \ | |
| CALC_F2_POST(DI,DX,CX,SI) | |
| #define CALC_112 \ | |
| CALC_F2_PRE(0x110,SI,DI,BX) \ | |
| PRECALC_32(Y8,Y7) \ | |
| CALC_F2_POST(SI,AX,DX,BX) | |
| #define CALC_113 \ | |
| CALC_F2_PRE(0x114,BX,SI,CX) \ | |
| PRECALC_33(Y3,Y5) \ | |
| CALC_F2_POST(BX,DI,AX,CX) | |
| #define CALC_114 \ | |
| CALC_F2_PRE(0x118,CX,BX,DX) \ | |
| PRECALC_34(Y13) \ | |
| CALC_F2_POST(CX,SI,DI,DX) | |
| #define CALC_115 \ | |
| CALC_F2_PRE(0x11c,DX,CX,AX) \ | |
| PRECALC_35(Y5) \ | |
| CALC_F2_POST(DX,BX,SI,AX) | |
| #define CALC_116 \ | |
| CALC_F2_PRE(0x130,AX,DX,DI) \ | |
| PRECALC_36(Y5) \ | |
| CALC_F2_POST(AX,CX,BX,DI) | |
| #define CALC_117 \ | |
| CALC_F2_PRE(0x134,DI,AX,SI) \ | |
| PRECALC_37(Y5) \ | |
| CALC_F2_POST(DI,DX,CX,SI) | |
| #define CALC_118 \ | |
| CALC_F2_PRE(0x138,SI,DI,BX) \ | |
| CALC_F2_POST(SI,AX,DX,BX) | |
| #define CALC_119 \ | |
| CALC_F3_PRE(0x13c,CX) \ | |
| PRECALC_39(Y5,0x40,0x1c0) \ | |
| CALC_F3_POST(BX,DI,AX,CX,SI) | |
| #define CALC_120 \ | |
| CALC_F3_PRE(0x150,DX) \ | |
| PRECALC_32(Y7,Y5) \ | |
| CALC_F3_POST(CX,SI,DI,DX,BX) | |
| #define CALC_121 \ | |
| CALC_F3_PRE(0x154,AX) \ | |
| PRECALC_33(Y15,Y3) \ | |
| CALC_F3_POST(DX,BX,SI,AX,CX) | |
| #define CALC_122 \ | |
| CALC_F3_PRE(0x158,DI) \ | |
| PRECALC_34(Y12) \ | |
| CALC_F3_POST(AX,CX,BX,DI,DX) | |
| #define CALC_123 \ | |
| CALC_F3_PRE(0x15c,SI) \ | |
| PRECALC_35(Y3) \ | |
| CALC_F3_POST(DI,DX,CX,SI,AX) | |
| #define CALC_124 \ | |
| CALC_F3_PRE(0x170,BX) \ | |
| PRECALC_36(Y3) \ | |
| CALC_F3_POST(SI,AX,DX,BX,DI) | |
| #define CALC_125 \ | |
| CALC_F3_PRE(0x174,CX) \ | |
| PRECALC_37(Y3) \ | |
| CALC_F3_POST(BX,DI,AX,CX,SI) | |
| #define CALC_126 \ | |
| CALC_F3_PRE(0x178,DX) \ | |
| CALC_F3_POST(CX,SI,DI,DX,BX) | |
| #define CALC_127 \ | |
| CALC_F3_PRE(0x17c,AX) \ | |
| PRECALC_39(Y3,0x60,0x1e0) \ | |
| CALC_F3_POST(DX,BX,SI,AX,CX) | |
| #define CALC_128 \ | |
| CALC_F3_PRE(0x190,DI) \ | |
| PRECALC_32(Y5,Y3) \ | |
| CALC_F3_POST(AX,CX,BX,DI,DX) | |
| #define CALC_129 \ | |
| CALC_F3_PRE(0x194,SI) \ | |
| PRECALC_33(Y14,Y15) \ | |
| CALC_F3_POST(DI,DX,CX,SI,AX) | |
| #define CALC_130 \ | |
| CALC_F3_PRE(0x198,BX) \ | |
| PRECALC_34(Y8) \ | |
| CALC_F3_POST(SI,AX,DX,BX,DI) | |
| #define CALC_131 \ | |
| CALC_F3_PRE(0x19c,CX) \ | |
| PRECALC_35(Y15) \ | |
| CALC_F3_POST(BX,DI,AX,CX,SI) | |
| #define CALC_132 \ | |
| CALC_F3_PRE(0x1b0,DX) \ | |
| PRECALC_36(Y15) \ | |
| CALC_F3_POST(CX,SI,DI,DX,BX) | |
| #define CALC_133 \ | |
| CALC_F3_PRE(0x1b4,AX) \ | |
| PRECALC_37(Y15) \ | |
| CALC_F3_POST(DX,BX,SI,AX,CX) | |
| #define CALC_134 \ | |
| CALC_F3_PRE(0x1b8,DI) \ | |
| CALC_F3_POST(AX,CX,BX,DI,DX) | |
| #define CALC_135 \ | |
| CALC_F3_PRE(0x1bc,SI) \ | |
| PRECALC_39(Y15,0x60,0x200) \ | |
| CALC_F3_POST(DI,DX,CX,SI,AX) | |
| #define CALC_136 \ | |
| CALC_F3_PRE(0x1d0,BX) \ | |
| PRECALC_32(Y3,Y15) \ | |
| CALC_F3_POST(SI,AX,DX,BX,DI) | |
| #define CALC_137 \ | |
| CALC_F3_PRE(0x1d4,CX) \ | |
| PRECALC_33(Y13,Y14) \ | |
| CALC_F3_POST(BX,DI,AX,CX,SI) | |
| #define CALC_138 \ | |
| CALC_F3_PRE(0x1d8,DX) \ | |
| PRECALC_34(Y7) \ | |
| CALC_F3_POST(CX,SI,DI,DX,BX) | |
| #define CALC_139 \ | |
| CALC_F2_PRE(0x1dc,DX,CX,AX) \ | |
| PRECALC_35(Y14) \ | |
| CALC_F2_POST(DX,BX,SI,AX) | |
| #define CALC_140 \ | |
| CALC_F2_PRE(0x1f0,AX,DX,DI) \ | |
| PRECALC_36(Y14) \ | |
| CALC_F2_POST(AX,CX,BX,DI) | |
| #define CALC_141 \ | |
| CALC_F2_PRE(0x1f4,DI,AX,SI) \ | |
| PRECALC_37(Y14) \ | |
| CALC_F2_POST(DI,DX,CX,SI) | |
| #define CALC_142 \ | |
| CALC_F2_PRE(0x1f8,SI,DI,BX) \ | |
| CALC_F2_POST(SI,AX,DX,BX) | |
| #define CALC_143 \ | |
| CALC_F2_PRE(0x1fc,BX,SI,CX) \ | |
| PRECALC_39(Y14,0x60,0x220) \ | |
| CALC_F2_POST(BX,DI,AX,CX) | |
| #define CALC_144 \ | |
| CALC_F2_PRE(0x210,CX,BX,DX) \ | |
| PRECALC_32(Y15,Y14) \ | |
| CALC_F2_POST(CX,SI,DI,DX) | |
| #define CALC_145 \ | |
| CALC_F2_PRE(0x214,DX,CX,AX) \ | |
| PRECALC_33(Y12,Y13) \ | |
| CALC_F2_POST(DX,BX,SI,AX) | |
| #define CALC_146 \ | |
| CALC_F2_PRE(0x218,AX,DX,DI) \ | |
| PRECALC_34(Y5) \ | |
| CALC_F2_POST(AX,CX,BX,DI) | |
| #define CALC_147 \ | |
| CALC_F2_PRE(0x21c,DI,AX,SI) \ | |
| PRECALC_35(Y13) \ | |
| CALC_F2_POST(DI,DX,CX,SI) | |
| #define CALC_148 \ | |
| CALC_F2_PRE(0x230,SI,DI,BX) \ | |
| PRECALC_36(Y13) \ | |
| CALC_F2_POST(SI,AX,DX,BX) | |
| #define CALC_149 \ | |
| CALC_F2_PRE(0x234,BX,SI,CX) \ | |
| PRECALC_37(Y13) \ | |
| CALC_F2_POST(BX,DI,AX,CX) | |
| #define CALC_150 \ | |
| CALC_F2_PRE(0x238,CX,BX,DX) \ | |
| CALC_F2_POST(CX,SI,DI,DX) | |
| #define CALC_151 \ | |
| CALC_F2_PRE(0x23c,DX,CX,AX) \ | |
| PRECALC_39(Y13,0x60,0x240) \ | |
| CALC_F2_POST(DX,BX,SI,AX) | |
| #define CALC_152 \ | |
| CALC_F2_PRE(0x250,AX,DX,DI) \ | |
| PRECALC_32(Y14,Y13) \ | |
| CALC_F2_POST(AX,CX,BX,DI) | |
| #define CALC_153 \ | |
| CALC_F2_PRE(0x254,DI,AX,SI) \ | |
| PRECALC_33(Y8,Y12) \ | |
| CALC_F2_POST(DI,DX,CX,SI) | |
| #define CALC_154 \ | |
| CALC_F2_PRE(0x258,SI,DI,BX) \ | |
| PRECALC_34(Y3) \ | |
| CALC_F2_POST(SI,AX,DX,BX) | |
| #define CALC_155 \ | |
| CALC_F2_PRE(0x25c,BX,SI,CX) \ | |
| PRECALC_35(Y12) \ | |
| CALC_F2_POST(BX,DI,AX,CX) | |
| #define CALC_156 \ | |
| CALC_F2_PRE(0x270,CX,BX,DX) \ | |
| PRECALC_36(Y12) \ | |
| CALC_F2_POST(CX,SI,DI,DX) | |
| #define CALC_157 \ | |
| CALC_F2_PRE(0x274,DX,CX,AX) \ | |
| PRECALC_37(Y12) \ | |
| CALC_F2_POST(DX,BX,SI,AX) | |
| #define CALC_158 \ | |
| CALC_F2_PRE(0x278,AX,DX,DI) \ | |
| CALC_F2_POST(AX,CX,BX,DI) | |
| #define CALC_159 \ | |
| ADDL 0x27c(R15),SI \ | |
| LEAL (SI)(AX*1), SI \ | |
| RORXL $0x1b, DI, R12 \ | |
| PRECALC_39(Y12,0x60,0x260) \ | |
| ADDL R12, SI | |
| #define CALC \ | |
| MOVL (R9), CX \ | |
| MOVL 4(R9), SI \ | |
| MOVL 8(R9), DI \ | |
| MOVL 12(R9), AX \ | |
| MOVL 16(R9), DX \ | |
| MOVQ SP, R14 \ | |
| LEAQ (2*4*80+32)(SP), R15 \ | |
| PRECALC \ // Precalc WK for first 2 blocks | |
| XCHGQ R15, R14 \ | |
| loop: \ // this loops is unrolled | |
| CMPQ R10, R8 \ // we use R8 value (set below) as a signal of a last block | |
| JNE begin \ | |
| VZEROUPPER \ | |
| RET \ | |
| begin: \ | |
| CALC_0 \ | |
| CALC_1 \ | |
| CALC_2 \ | |
| CALC_3 \ | |
| CALC_4 \ | |
| CALC_5 \ | |
| CALC_6 \ | |
| CALC_7 \ | |
| CALC_8 \ | |
| CALC_9 \ | |
| CALC_10 \ | |
| CALC_11 \ | |
| CALC_12 \ | |
| CALC_13 \ | |
| CALC_14 \ | |
| CALC_15 \ | |
| CALC_16 \ | |
| CALC_17 \ | |
| CALC_18 \ | |
| CALC_19 \ | |
| CALC_20 \ | |
| CALC_21 \ | |
| CALC_22 \ | |
| CALC_23 \ | |
| CALC_24 \ | |
| CALC_25 \ | |
| CALC_26 \ | |
| CALC_27 \ | |
| CALC_28 \ | |
| CALC_29 \ | |
| CALC_30 \ | |
| CALC_31 \ | |
| CALC_32 \ | |
| CALC_33 \ | |
| CALC_34 \ | |
| CALC_35 \ | |
| CALC_36 \ | |
| CALC_37 \ | |
| CALC_38 \ | |
| CALC_39 \ | |
| CALC_40 \ | |
| CALC_41 \ | |
| CALC_42 \ | |
| CALC_43 \ | |
| CALC_44 \ | |
| CALC_45 \ | |
| CALC_46 \ | |
| CALC_47 \ | |
| CALC_48 \ | |
| CALC_49 \ | |
| CALC_50 \ | |
| CALC_51 \ | |
| CALC_52 \ | |
| CALC_53 \ | |
| CALC_54 \ | |
| CALC_55 \ | |
| CALC_56 \ | |
| CALC_57 \ | |
| CALC_58 \ | |
| CALC_59 \ | |
| ADDQ $128, R10 \ // move to next even-64-byte block | |
| CMPQ R10, R11 \ // is current block the last one? | |
| CMOVQCC R8, R10 \ // signal the last iteration smartly | |
| CALC_60 \ | |
| CALC_61 \ | |
| CALC_62 \ | |
| CALC_63 \ | |
| CALC_64 \ | |
| CALC_65 \ | |
| CALC_66 \ | |
| CALC_67 \ | |
| CALC_68 \ | |
| CALC_69 \ | |
| CALC_70 \ | |
| CALC_71 \ | |
| CALC_72 \ | |
| CALC_73 \ | |
| CALC_74 \ | |
| CALC_75 \ | |
| CALC_76 \ | |
| CALC_77 \ | |
| CALC_78 \ | |
| CALC_79 \ | |
| UPDATE_HASH(AX,DX,BX,SI,DI) \ | |
| CMPQ R10, R8 \ // is current block the last one? | |
| JE loop\ | |
| MOVL DX, CX \ | |
| CALC_80 \ | |
| CALC_81 \ | |
| CALC_82 \ | |
| CALC_83 \ | |
| CALC_84 \ | |
| CALC_85 \ | |
| CALC_86 \ | |
| CALC_87 \ | |
| CALC_88 \ | |
| CALC_89 \ | |
| CALC_90 \ | |
| CALC_91 \ | |
| CALC_92 \ | |
| CALC_93 \ | |
| CALC_94 \ | |
| CALC_95 \ | |
| CALC_96 \ | |
| CALC_97 \ | |
| CALC_98 \ | |
| CALC_99 \ | |
| CALC_100 \ | |
| CALC_101 \ | |
| CALC_102 \ | |
| CALC_103 \ | |
| CALC_104 \ | |
| CALC_105 \ | |
| CALC_106 \ | |
| CALC_107 \ | |
| CALC_108 \ | |
| CALC_109 \ | |
| CALC_110 \ | |
| CALC_111 \ | |
| CALC_112 \ | |
| CALC_113 \ | |
| CALC_114 \ | |
| CALC_115 \ | |
| CALC_116 \ | |
| CALC_117 \ | |
| CALC_118 \ | |
| CALC_119 \ | |
| CALC_120 \ | |
| CALC_121 \ | |
| CALC_122 \ | |
| CALC_123 \ | |
| CALC_124 \ | |
| CALC_125 \ | |
| CALC_126 \ | |
| CALC_127 \ | |
| CALC_128 \ | |
| CALC_129 \ | |
| CALC_130 \ | |
| CALC_131 \ | |
| CALC_132 \ | |
| CALC_133 \ | |
| CALC_134 \ | |
| CALC_135 \ | |
| CALC_136 \ | |
| CALC_137 \ | |
| CALC_138 \ | |
| CALC_139 \ | |
| ADDQ $128, R13 \ //move to next even-64-byte block | |
| CMPQ R13, R11 \ //is current block the last one? | |
| CMOVQCC R8, R10 \ | |
| CALC_140 \ | |
| CALC_141 \ | |
| CALC_142 \ | |
| CALC_143 \ | |
| CALC_144 \ | |
| CALC_145 \ | |
| CALC_146 \ | |
| CALC_147 \ | |
| CALC_148 \ | |
| CALC_149 \ | |
| CALC_150 \ | |
| CALC_151 \ | |
| CALC_152 \ | |
| CALC_153 \ | |
| CALC_154 \ | |
| CALC_155 \ | |
| CALC_156 \ | |
| CALC_157 \ | |
| CALC_158 \ | |
| CALC_159 \ | |
| UPDATE_HASH(SI,DI,DX,CX,BX) \ | |
| MOVL SI, R12 \ //Reset state for AVX2 reg permutation | |
| MOVL DI, SI \ | |
| MOVL DX, DI \ | |
| MOVL BX, DX \ | |
| MOVL CX, AX \ | |
| MOVL R12, CX \ | |
| XCHGQ R15, R14 \ | |
| JMP loop | |
| TEXT ·blockAVX2(SB),$1408-32 | |
| MOVQ dig+0(FP), DI | |
| MOVQ p_base+8(FP), SI | |
| MOVQ p_len+16(FP), DX | |
| SHRQ $6, DX | |
| SHLQ $6, DX | |
| MOVQ $K_XMM_AR<>(SB), R8 | |
| MOVQ DI, R9 | |
| MOVQ SI, R10 | |
| LEAQ 64(SI), R13 | |
| ADDQ SI, DX | |
| ADDQ $64, DX | |
| MOVQ DX, R11 | |
| CMPQ R13, R11 | |
| CMOVQCC R8, R13 | |
| VMOVDQU BSWAP_SHUFB_CTL<>(SB), Y10 | |
| CALC // RET is inside macros | |
| DATA K_XMM_AR<>+0x00(SB)/4,$0x5a827999 | |
| DATA K_XMM_AR<>+0x04(SB)/4,$0x5a827999 | |
| DATA K_XMM_AR<>+0x08(SB)/4,$0x5a827999 | |
| DATA K_XMM_AR<>+0x0c(SB)/4,$0x5a827999 | |
| DATA K_XMM_AR<>+0x10(SB)/4,$0x5a827999 | |
| DATA K_XMM_AR<>+0x14(SB)/4,$0x5a827999 | |
| DATA K_XMM_AR<>+0x18(SB)/4,$0x5a827999 | |
| DATA K_XMM_AR<>+0x1c(SB)/4,$0x5a827999 | |
| DATA K_XMM_AR<>+0x20(SB)/4,$0x6ed9eba1 | |
| DATA K_XMM_AR<>+0x24(SB)/4,$0x6ed9eba1 | |
| DATA K_XMM_AR<>+0x28(SB)/4,$0x6ed9eba1 | |
| DATA K_XMM_AR<>+0x2c(SB)/4,$0x6ed9eba1 | |
| DATA K_XMM_AR<>+0x30(SB)/4,$0x6ed9eba1 | |
| DATA K_XMM_AR<>+0x34(SB)/4,$0x6ed9eba1 | |
| DATA K_XMM_AR<>+0x38(SB)/4,$0x6ed9eba1 | |
| DATA K_XMM_AR<>+0x3c(SB)/4,$0x6ed9eba1 | |
| DATA K_XMM_AR<>+0x40(SB)/4,$0x8f1bbcdc | |
| DATA K_XMM_AR<>+0x44(SB)/4,$0x8f1bbcdc | |
| DATA K_XMM_AR<>+0x48(SB)/4,$0x8f1bbcdc | |
| DATA K_XMM_AR<>+0x4c(SB)/4,$0x8f1bbcdc | |
| DATA K_XMM_AR<>+0x50(SB)/4,$0x8f1bbcdc | |
| DATA K_XMM_AR<>+0x54(SB)/4,$0x8f1bbcdc | |
| DATA K_XMM_AR<>+0x58(SB)/4,$0x8f1bbcdc | |
| DATA K_XMM_AR<>+0x5c(SB)/4,$0x8f1bbcdc | |
| DATA K_XMM_AR<>+0x60(SB)/4,$0xca62c1d6 | |
| DATA K_XMM_AR<>+0x64(SB)/4,$0xca62c1d6 | |
| DATA K_XMM_AR<>+0x68(SB)/4,$0xca62c1d6 | |
| DATA K_XMM_AR<>+0x6c(SB)/4,$0xca62c1d6 | |
| DATA K_XMM_AR<>+0x70(SB)/4,$0xca62c1d6 | |
| DATA K_XMM_AR<>+0x74(SB)/4,$0xca62c1d6 | |
| DATA K_XMM_AR<>+0x78(SB)/4,$0xca62c1d6 | |
| DATA K_XMM_AR<>+0x7c(SB)/4,$0xca62c1d6 | |
| GLOBL K_XMM_AR<>(SB),RODATA,$128 | |
| DATA BSWAP_SHUFB_CTL<>+0x00(SB)/4,$0x00010203 | |
| DATA BSWAP_SHUFB_CTL<>+0x04(SB)/4,$0x04050607 | |
| DATA BSWAP_SHUFB_CTL<>+0x08(SB)/4,$0x08090a0b | |
| DATA BSWAP_SHUFB_CTL<>+0x0c(SB)/4,$0x0c0d0e0f | |
| DATA BSWAP_SHUFB_CTL<>+0x10(SB)/4,$0x00010203 | |
| DATA BSWAP_SHUFB_CTL<>+0x14(SB)/4,$0x04050607 | |
| DATA BSWAP_SHUFB_CTL<>+0x18(SB)/4,$0x08090a0b | |
| DATA BSWAP_SHUFB_CTL<>+0x1c(SB)/4,$0x0c0d0e0f | |
| GLOBL BSWAP_SHUFB_CTL<>(SB),RODATA,$32 |