| // Copyright 2013 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| // AVX2 version by Intel, same algorithm as code in Linux kernel: |
| // https://github.com/torvalds/linux/blob/master/arch/x86/crypto/sha1_avx2_x86_64_asm.S |
| // Authors: |
| // Ilya Albrekht <ilya.albrekht@intel.com> |
| // Maxim Locktyukhin <maxim.locktyukhin@intel.com> |
| // Ronen Zohar <ronen.zohar@intel.com> |
| // Chandramouli Narayanan <mouli@linux.intel.com> |
| |
| |
| #include "textflag.h" |
| |
| // SHA-1 block routine. See sha1block.go for Go equivalent. |
| // |
| // There are 80 rounds of 4 types: |
| // - rounds 0-15 are type 1 and load data (ROUND1 macro). |
| // - rounds 16-19 are type 1 and do not load data (ROUND1x macro). |
| // - rounds 20-39 are type 2 and do not load data (ROUND2 macro). |
| // - rounds 40-59 are type 3 and do not load data (ROUND3 macro). |
| // - rounds 60-79 are type 4 and do not load data (ROUND4 macro). |
| // |
| // Each round loads or shuffles the data, then computes a per-round |
| // function of b, c, d, and then mixes the result into and rotates the |
| // five registers a, b, c, d, e holding the intermediate results. |
| // |
| // The register rotation is implemented by rotating the arguments to |
| // the round macros instead of by explicit move instructions. |
| |
| #define LOAD(index) \ |
| MOVL (index*4)(SI), R10; \ |
| BSWAPL R10; \ |
| MOVL R10, (index*4)(SP) |
| |
| #define SHUFFLE(index) \ |
| MOVL (((index)&0xf)*4)(SP), R10; \ |
| XORL (((index-3)&0xf)*4)(SP), R10; \ |
| XORL (((index-8)&0xf)*4)(SP), R10; \ |
| XORL (((index-14)&0xf)*4)(SP), R10; \ |
| ROLL $1, R10; \ |
| MOVL R10, (((index)&0xf)*4)(SP) |
| |
| #define FUNC1(a, b, c, d, e) \ |
| MOVL d, R9; \ |
| XORL c, R9; \ |
| ANDL b, R9; \ |
| XORL d, R9 |
| |
| #define FUNC2(a, b, c, d, e) \ |
| MOVL b, R9; \ |
| XORL c, R9; \ |
| XORL d, R9 |
| |
| #define FUNC3(a, b, c, d, e) \ |
| MOVL b, R8; \ |
| ORL c, R8; \ |
| ANDL d, R8; \ |
| MOVL b, R9; \ |
| ANDL c, R9; \ |
| ORL R8, R9 |
| |
| #define FUNC4 FUNC2 |
| |
| #define MIX(a, b, c, d, e, const) \ |
| ROLL $30, b; \ |
| ADDL R9, e; \ |
| MOVL a, R8; \ |
| ROLL $5, R8; \ |
| LEAL const(e)(R10*1), e; \ |
| ADDL R8, e |
| |
| #define ROUND1(a, b, c, d, e, index) \ |
| LOAD(index); \ |
| FUNC1(a, b, c, d, e); \ |
| MIX(a, b, c, d, e, 0x5A827999) |
| |
| #define ROUND1x(a, b, c, d, e, index) \ |
| SHUFFLE(index); \ |
| FUNC1(a, b, c, d, e); \ |
| MIX(a, b, c, d, e, 0x5A827999) |
| |
| #define ROUND2(a, b, c, d, e, index) \ |
| SHUFFLE(index); \ |
| FUNC2(a, b, c, d, e); \ |
| MIX(a, b, c, d, e, 0x6ED9EBA1) |
| |
| #define ROUND3(a, b, c, d, e, index) \ |
| SHUFFLE(index); \ |
| FUNC3(a, b, c, d, e); \ |
| MIX(a, b, c, d, e, 0x8F1BBCDC) |
| |
| #define ROUND4(a, b, c, d, e, index) \ |
| SHUFFLE(index); \ |
| FUNC4(a, b, c, d, e); \ |
| MIX(a, b, c, d, e, 0xCA62C1D6) |
| |
| TEXT ·blockAMD64(SB),NOSPLIT,$64-32 |
| MOVQ dig+0(FP), BP |
| MOVQ p_base+8(FP), SI |
| MOVQ p_len+16(FP), DX |
| SHRQ $6, DX |
| SHLQ $6, DX |
| |
| LEAQ (SI)(DX*1), DI |
| MOVL (0*4)(BP), AX |
| MOVL (1*4)(BP), BX |
| MOVL (2*4)(BP), CX |
| MOVL (3*4)(BP), DX |
| MOVL (4*4)(BP), BP |
| |
| CMPQ SI, DI |
| JEQ end |
| |
| loop: |
| MOVL AX, R11 |
| MOVL BX, R12 |
| MOVL CX, R13 |
| MOVL DX, R14 |
| MOVL BP, R15 |
| |
| ROUND1(AX, BX, CX, DX, BP, 0) |
| ROUND1(BP, AX, BX, CX, DX, 1) |
| ROUND1(DX, BP, AX, BX, CX, 2) |
| ROUND1(CX, DX, BP, AX, BX, 3) |
| ROUND1(BX, CX, DX, BP, AX, 4) |
| ROUND1(AX, BX, CX, DX, BP, 5) |
| ROUND1(BP, AX, BX, CX, DX, 6) |
| ROUND1(DX, BP, AX, BX, CX, 7) |
| ROUND1(CX, DX, BP, AX, BX, 8) |
| ROUND1(BX, CX, DX, BP, AX, 9) |
| ROUND1(AX, BX, CX, DX, BP, 10) |
| ROUND1(BP, AX, BX, CX, DX, 11) |
| ROUND1(DX, BP, AX, BX, CX, 12) |
| ROUND1(CX, DX, BP, AX, BX, 13) |
| ROUND1(BX, CX, DX, BP, AX, 14) |
| ROUND1(AX, BX, CX, DX, BP, 15) |
| |
| ROUND1x(BP, AX, BX, CX, DX, 16) |
| ROUND1x(DX, BP, AX, BX, CX, 17) |
| ROUND1x(CX, DX, BP, AX, BX, 18) |
| ROUND1x(BX, CX, DX, BP, AX, 19) |
| |
| ROUND2(AX, BX, CX, DX, BP, 20) |
| ROUND2(BP, AX, BX, CX, DX, 21) |
| ROUND2(DX, BP, AX, BX, CX, 22) |
| ROUND2(CX, DX, BP, AX, BX, 23) |
| ROUND2(BX, CX, DX, BP, AX, 24) |
| ROUND2(AX, BX, CX, DX, BP, 25) |
| ROUND2(BP, AX, BX, CX, DX, 26) |
| ROUND2(DX, BP, AX, BX, CX, 27) |
| ROUND2(CX, DX, BP, AX, BX, 28) |
| ROUND2(BX, CX, DX, BP, AX, 29) |
| ROUND2(AX, BX, CX, DX, BP, 30) |
| ROUND2(BP, AX, BX, CX, DX, 31) |
| ROUND2(DX, BP, AX, BX, CX, 32) |
| ROUND2(CX, DX, BP, AX, BX, 33) |
| ROUND2(BX, CX, DX, BP, AX, 34) |
| ROUND2(AX, BX, CX, DX, BP, 35) |
| ROUND2(BP, AX, BX, CX, DX, 36) |
| ROUND2(DX, BP, AX, BX, CX, 37) |
| ROUND2(CX, DX, BP, AX, BX, 38) |
| ROUND2(BX, CX, DX, BP, AX, 39) |
| |
| ROUND3(AX, BX, CX, DX, BP, 40) |
| ROUND3(BP, AX, BX, CX, DX, 41) |
| ROUND3(DX, BP, AX, BX, CX, 42) |
| ROUND3(CX, DX, BP, AX, BX, 43) |
| ROUND3(BX, CX, DX, BP, AX, 44) |
| ROUND3(AX, BX, CX, DX, BP, 45) |
| ROUND3(BP, AX, BX, CX, DX, 46) |
| ROUND3(DX, BP, AX, BX, CX, 47) |
| ROUND3(CX, DX, BP, AX, BX, 48) |
| ROUND3(BX, CX, DX, BP, AX, 49) |
| ROUND3(AX, BX, CX, DX, BP, 50) |
| ROUND3(BP, AX, BX, CX, DX, 51) |
| ROUND3(DX, BP, AX, BX, CX, 52) |
| ROUND3(CX, DX, BP, AX, BX, 53) |
| ROUND3(BX, CX, DX, BP, AX, 54) |
| ROUND3(AX, BX, CX, DX, BP, 55) |
| ROUND3(BP, AX, BX, CX, DX, 56) |
| ROUND3(DX, BP, AX, BX, CX, 57) |
| ROUND3(CX, DX, BP, AX, BX, 58) |
| ROUND3(BX, CX, DX, BP, AX, 59) |
| |
| ROUND4(AX, BX, CX, DX, BP, 60) |
| ROUND4(BP, AX, BX, CX, DX, 61) |
| ROUND4(DX, BP, AX, BX, CX, 62) |
| ROUND4(CX, DX, BP, AX, BX, 63) |
| ROUND4(BX, CX, DX, BP, AX, 64) |
| ROUND4(AX, BX, CX, DX, BP, 65) |
| ROUND4(BP, AX, BX, CX, DX, 66) |
| ROUND4(DX, BP, AX, BX, CX, 67) |
| ROUND4(CX, DX, BP, AX, BX, 68) |
| ROUND4(BX, CX, DX, BP, AX, 69) |
| ROUND4(AX, BX, CX, DX, BP, 70) |
| ROUND4(BP, AX, BX, CX, DX, 71) |
| ROUND4(DX, BP, AX, BX, CX, 72) |
| ROUND4(CX, DX, BP, AX, BX, 73) |
| ROUND4(BX, CX, DX, BP, AX, 74) |
| ROUND4(AX, BX, CX, DX, BP, 75) |
| ROUND4(BP, AX, BX, CX, DX, 76) |
| ROUND4(DX, BP, AX, BX, CX, 77) |
| ROUND4(CX, DX, BP, AX, BX, 78) |
| ROUND4(BX, CX, DX, BP, AX, 79) |
| |
| ADDL R11, AX |
| ADDL R12, BX |
| ADDL R13, CX |
| ADDL R14, DX |
| ADDL R15, BP |
| |
| ADDQ $64, SI |
| CMPQ SI, DI |
| JB loop |
| |
| end: |
| MOVQ dig+0(FP), DI |
| MOVL AX, (0*4)(DI) |
| MOVL BX, (1*4)(DI) |
| MOVL CX, (2*4)(DI) |
| MOVL DX, (3*4)(DI) |
| MOVL BP, (4*4)(DI) |
| RET |
| |
| |
| // This is the implementation using AVX2, BMI1 and BMI2. It is based on: |
| // "SHA-1 implementation with Intel(R) AVX2 instruction set extensions" |
| // From http://software.intel.com/en-us/articles |
| // (look for improving-the-performance-of-the-secure-hash-algorithm-1) |
| // This implementation is 2x unrolled, and interleaves vector instructions, |
| // used to precompute W, with scalar computation of current round |
| // for optimal scheduling. |
| |
| // Trivial helper macros. |
| #define UPDATE_HASH(A,TB,C,D,E) \ |
| ADDL (R9), A \ |
| MOVL A, (R9) \ |
| ADDL 4(R9), TB \ |
| MOVL TB, 4(R9) \ |
| ADDL 8(R9), C \ |
| MOVL C, 8(R9) \ |
| ADDL 12(R9), D \ |
| MOVL D, 12(R9) \ |
| ADDL 16(R9), E \ |
| MOVL E, 16(R9) |
| |
| |
| |
| // Helper macros for PRECALC, which does precomputations |
| #define PRECALC_0(OFFSET) \ |
| VMOVDQU OFFSET(R10),X0 |
| |
| #define PRECALC_1(OFFSET) \ |
| VINSERTI128 $1, OFFSET(R13), Y0, Y0 |
| |
| #define PRECALC_2(YREG) \ |
| VPSHUFB Y10, Y0, YREG |
| |
| #define PRECALC_4(YREG,K_OFFSET) \ |
| VPADDD K_OFFSET(R8), YREG, Y0 |
| |
| #define PRECALC_7(OFFSET) \ |
| VMOVDQU Y0, (OFFSET*2)(R14) |
| |
| |
| // Message scheduling pre-compute for rounds 0-15 |
| // R13 is a pointer to even 64-byte block |
| // R10 is a pointer to odd 64-byte block |
| // R14 is a pointer to temp buffer |
| // X0 is used as temp register |
| // YREG is clobbered as part of computation |
| // OFFSET chooses 16 byte chunk within a block |
| // R8 is a pointer to constants block |
| // K_OFFSET chooses K constants relevant to this round |
| // X10 holds swap mask |
| #define PRECALC_00_15(OFFSET,YREG) \ |
| PRECALC_0(OFFSET) \ |
| PRECALC_1(OFFSET) \ |
| PRECALC_2(YREG) \ |
| PRECALC_4(YREG,0x0) \ |
| PRECALC_7(OFFSET) |
| |
| |
| // Helper macros for PRECALC_16_31 |
| #define PRECALC_16(REG_SUB_16,REG_SUB_12,REG_SUB_4,REG) \ |
| VPALIGNR $8, REG_SUB_16, REG_SUB_12, REG \ // w[i-14] |
| VPSRLDQ $4, REG_SUB_4, Y0 // w[i-3] |
| |
| #define PRECALC_17(REG_SUB_16,REG_SUB_8,REG) \ |
| VPXOR REG_SUB_8, REG, REG \ |
| VPXOR REG_SUB_16, Y0, Y0 |
| |
| #define PRECALC_18(REG) \ |
| VPXOR Y0, REG, REG \ |
| VPSLLDQ $12, REG, Y9 |
| |
| #define PRECALC_19(REG) \ |
| VPSLLD $1, REG, Y0 \ |
| VPSRLD $31, REG, REG |
| |
| #define PRECALC_20(REG) \ |
| VPOR REG, Y0, Y0 \ |
| VPSLLD $2, Y9, REG |
| |
| #define PRECALC_21(REG) \ |
| VPSRLD $30, Y9, Y9 \ |
| VPXOR REG, Y0, Y0 |
| |
| #define PRECALC_23(REG,K_OFFSET,OFFSET) \ |
| VPXOR Y9, Y0, REG \ |
| VPADDD K_OFFSET(R8), REG, Y0 \ |
| VMOVDQU Y0, (OFFSET)(R14) |
| |
| // Message scheduling pre-compute for rounds 16-31 |
| // calculating last 32 w[i] values in 8 XMM registers |
| // pre-calculate K+w[i] values and store to mem |
| // for later load by ALU add instruction. |
| // "brute force" vectorization for rounds 16-31 only |
| // due to w[i]->w[i-3] dependency. |
| // clobbers 5 input ymm registers REG_SUB* |
| // uses X0 and X9 as temp registers |
| // As always, R8 is a pointer to constants block |
| // and R14 is a pointer to temp buffer |
| #define PRECALC_16_31(REG,REG_SUB_4,REG_SUB_8,REG_SUB_12,REG_SUB_16,K_OFFSET,OFFSET) \ |
| PRECALC_16(REG_SUB_16,REG_SUB_12,REG_SUB_4,REG) \ |
| PRECALC_17(REG_SUB_16,REG_SUB_8,REG) \ |
| PRECALC_18(REG) \ |
| PRECALC_19(REG) \ |
| PRECALC_20(REG) \ |
| PRECALC_21(REG) \ |
| PRECALC_23(REG,K_OFFSET,OFFSET) |
| |
| |
| // Helper macros for PRECALC_32_79 |
| #define PRECALC_32(REG_SUB_8,REG_SUB_4) \ |
| VPALIGNR $8, REG_SUB_8, REG_SUB_4, Y0 |
| |
| #define PRECALC_33(REG_SUB_28,REG) \ |
| VPXOR REG_SUB_28, REG, REG |
| |
| #define PRECALC_34(REG_SUB_16) \ |
| VPXOR REG_SUB_16, Y0, Y0 |
| |
| #define PRECALC_35(REG) \ |
| VPXOR Y0, REG, REG |
| |
| #define PRECALC_36(REG) \ |
| VPSLLD $2, REG, Y0 |
| |
| #define PRECALC_37(REG) \ |
| VPSRLD $30, REG, REG \ |
| VPOR REG, Y0, REG |
| |
| #define PRECALC_39(REG,K_OFFSET,OFFSET) \ |
| VPADDD K_OFFSET(R8), REG, Y0 \ |
| VMOVDQU Y0, (OFFSET)(R14) |
| |
| // Message scheduling pre-compute for rounds 32-79 |
| // In SHA-1 specification we have: |
| // w[i] = (w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16]) rol 1 |
| // Which is the same as: |
| // w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2 |
| // This allows for more efficient vectorization, |
| // since w[i]->w[i-3] dependency is broken |
| #define PRECALC_32_79(REG,REG_SUB_4,REG_SUB_8,REG_SUB_16,REG_SUB_28,K_OFFSET,OFFSET) \ |
| PRECALC_32(REG_SUB_8,REG_SUB_4) \ |
| PRECALC_33(REG_SUB_28,REG) \ |
| PRECALC_34(REG_SUB_16) \ |
| PRECALC_35(REG) \ |
| PRECALC_36(REG) \ |
| PRECALC_37(REG) \ |
| PRECALC_39(REG,K_OFFSET,OFFSET) |
| |
| #define PRECALC \ |
| PRECALC_00_15(0,Y15) \ |
| PRECALC_00_15(0x10,Y14) \ |
| PRECALC_00_15(0x20,Y13) \ |
| PRECALC_00_15(0x30,Y12) \ |
| PRECALC_16_31(Y8,Y12,Y13,Y14,Y15,0,0x80) \ |
| PRECALC_16_31(Y7,Y8,Y12,Y13,Y14,0x20,0xa0) \ |
| PRECALC_16_31(Y5,Y7,Y8,Y12,Y13,0x20,0xc0) \ |
| PRECALC_16_31(Y3,Y5,Y7,Y8,Y12,0x20,0xe0) \ |
| PRECALC_32_79(Y15,Y3,Y5,Y8,Y14,0x20,0x100) \ |
| PRECALC_32_79(Y14,Y15,Y3,Y7,Y13,0x20,0x120) \ |
| PRECALC_32_79(Y13,Y14,Y15,Y5,Y12,0x40,0x140) \ |
| PRECALC_32_79(Y12,Y13,Y14,Y3,Y8,0x40,0x160) \ |
| PRECALC_32_79(Y8,Y12,Y13,Y15,Y7,0x40,0x180) \ |
| PRECALC_32_79(Y7,Y8,Y12,Y14,Y5,0x40,0x1a0) \ |
| PRECALC_32_79(Y5,Y7,Y8,Y13,Y3,0x40,0x1c0) \ |
| PRECALC_32_79(Y3,Y5,Y7,Y12,Y15,0x60,0x1e0) \ |
| PRECALC_32_79(Y15,Y3,Y5,Y8,Y14,0x60,0x200) \ |
| PRECALC_32_79(Y14,Y15,Y3,Y7,Y13,0x60,0x220) \ |
| PRECALC_32_79(Y13,Y14,Y15,Y5,Y12,0x60,0x240) \ |
| PRECALC_32_79(Y12,Y13,Y14,Y3,Y8,0x60,0x260) |
| |
| // Macros calculating individual rounds have general form |
| // CALC_ROUND_PRE + PRECALC_ROUND + CALC_ROUND_POST |
| // CALC_ROUND_{PRE,POST} macros follow |
| |
| #define CALC_F1_PRE(OFFSET,REG_A,REG_B,REG_C,REG_E) \ |
| ADDL OFFSET(R15),REG_E \ |
| ANDNL REG_C,REG_A,BP \ |
| LEAL (REG_E)(REG_B*1), REG_E \ // Add F from the previous round |
| RORXL $0x1b, REG_A, R12 \ |
| RORXL $2, REG_A, REG_B // for next round |
| |
| // Calculate F for the next round |
| #define CALC_F1_POST(REG_A,REG_B,REG_E) \ |
| ANDL REG_B,REG_A \ // b&c |
| XORL BP, REG_A \ // F1 = (b&c) ^ (~b&d) |
| LEAL (REG_E)(R12*1), REG_E // E += A >>> 5 |
| |
| |
| // Registers are cyclically rotated DX -> AX -> DI -> SI -> BX -> CX |
| #define CALC_0 \ |
| MOVL SI, BX \ // Precalculating first round |
| RORXL $2, SI, SI \ |
| ANDNL AX, BX, BP \ |
| ANDL DI, BX \ |
| XORL BP, BX \ |
| CALC_F1_PRE(0x0,CX,BX,DI,DX) \ |
| PRECALC_0(0x80) \ |
| CALC_F1_POST(CX,SI,DX) |
| |
| #define CALC_1 \ |
| CALC_F1_PRE(0x4,DX,CX,SI,AX) \ |
| PRECALC_1(0x80) \ |
| CALC_F1_POST(DX,BX,AX) |
| |
| #define CALC_2 \ |
| CALC_F1_PRE(0x8,AX,DX,BX,DI) \ |
| PRECALC_2(Y15) \ |
| CALC_F1_POST(AX,CX,DI) |
| |
| #define CALC_3 \ |
| CALC_F1_PRE(0xc,DI,AX,CX,SI) \ |
| CALC_F1_POST(DI,DX,SI) |
| |
| #define CALC_4 \ |
| CALC_F1_PRE(0x20,SI,DI,DX,BX) \ |
| PRECALC_4(Y15,0x0) \ |
| CALC_F1_POST(SI,AX,BX) |
| |
| #define CALC_5 \ |
| CALC_F1_PRE(0x24,BX,SI,AX,CX) \ |
| CALC_F1_POST(BX,DI,CX) |
| |
| #define CALC_6 \ |
| CALC_F1_PRE(0x28,CX,BX,DI,DX) \ |
| CALC_F1_POST(CX,SI,DX) |
| |
| #define CALC_7 \ |
| CALC_F1_PRE(0x2c,DX,CX,SI,AX) \ |
| PRECALC_7(0x0) \ |
| CALC_F1_POST(DX,BX,AX) |
| |
| #define CALC_8 \ |
| CALC_F1_PRE(0x40,AX,DX,BX,DI) \ |
| PRECALC_0(0x90) \ |
| CALC_F1_POST(AX,CX,DI) |
| |
| #define CALC_9 \ |
| CALC_F1_PRE(0x44,DI,AX,CX,SI) \ |
| PRECALC_1(0x90) \ |
| CALC_F1_POST(DI,DX,SI) |
| |
| #define CALC_10 \ |
| CALC_F1_PRE(0x48,SI,DI,DX,BX) \ |
| PRECALC_2(Y14) \ |
| CALC_F1_POST(SI,AX,BX) |
| |
| #define CALC_11 \ |
| CALC_F1_PRE(0x4c,BX,SI,AX,CX) \ |
| CALC_F1_POST(BX,DI,CX) |
| |
| #define CALC_12 \ |
| CALC_F1_PRE(0x60,CX,BX,DI,DX) \ |
| PRECALC_4(Y14,0x0) \ |
| CALC_F1_POST(CX,SI,DX) |
| |
| #define CALC_13 \ |
| CALC_F1_PRE(0x64,DX,CX,SI,AX) \ |
| CALC_F1_POST(DX,BX,AX) |
| |
| #define CALC_14 \ |
| CALC_F1_PRE(0x68,AX,DX,BX,DI) \ |
| CALC_F1_POST(AX,CX,DI) |
| |
| #define CALC_15 \ |
| CALC_F1_PRE(0x6c,DI,AX,CX,SI) \ |
| PRECALC_7(0x10) \ |
| CALC_F1_POST(DI,DX,SI) |
| |
| #define CALC_16 \ |
| CALC_F1_PRE(0x80,SI,DI,DX,BX) \ |
| PRECALC_0(0xa0) \ |
| CALC_F1_POST(SI,AX,BX) |
| |
| #define CALC_17 \ |
| CALC_F1_PRE(0x84,BX,SI,AX,CX) \ |
| PRECALC_1(0xa0) \ |
| CALC_F1_POST(BX,DI,CX) |
| |
| #define CALC_18 \ |
| CALC_F1_PRE(0x88,CX,BX,DI,DX) \ |
| PRECALC_2(Y13) \ |
| CALC_F1_POST(CX,SI,DX) |
| |
| |
| #define CALC_F2_PRE(OFFSET,REG_A,REG_B,REG_E) \ |
| ADDL OFFSET(R15),REG_E \ |
| LEAL (REG_E)(REG_B*1), REG_E \ // Add F from the previous round |
| RORXL $0x1b, REG_A, R12 \ |
| RORXL $2, REG_A, REG_B // for next round |
| |
| #define CALC_F2_POST(REG_A,REG_B,REG_C,REG_E) \ |
| XORL REG_B, REG_A \ |
| ADDL R12, REG_E \ |
| XORL REG_C, REG_A |
| |
| #define CALC_19 \ |
| CALC_F2_PRE(0x8c,DX,CX,AX) \ |
| CALC_F2_POST(DX,BX,SI,AX) |
| |
| #define CALC_20 \ |
| CALC_F2_PRE(0xa0,AX,DX,DI) \ |
| PRECALC_4(Y13,0x0) \ |
| CALC_F2_POST(AX,CX,BX,DI) |
| |
| #define CALC_21 \ |
| CALC_F2_PRE(0xa4,DI,AX,SI) \ |
| CALC_F2_POST(DI,DX,CX,SI) |
| |
| #define CALC_22 \ |
| CALC_F2_PRE(0xa8,SI,DI,BX) \ |
| CALC_F2_POST(SI,AX,DX,BX) |
| |
| #define CALC_23 \ |
| CALC_F2_PRE(0xac,BX,SI,CX) \ |
| PRECALC_7(0x20) \ |
| CALC_F2_POST(BX,DI,AX,CX) |
| |
| #define CALC_24 \ |
| CALC_F2_PRE(0xc0,CX,BX,DX) \ |
| PRECALC_0(0xb0) \ |
| CALC_F2_POST(CX,SI,DI,DX) |
| |
| #define CALC_25 \ |
| CALC_F2_PRE(0xc4,DX,CX,AX) \ |
| PRECALC_1(0xb0) \ |
| CALC_F2_POST(DX,BX,SI,AX) |
| |
| #define CALC_26 \ |
| CALC_F2_PRE(0xc8,AX,DX,DI) \ |
| PRECALC_2(Y12) \ |
| CALC_F2_POST(AX,CX,BX,DI) |
| |
| #define CALC_27 \ |
| CALC_F2_PRE(0xcc,DI,AX,SI) \ |
| CALC_F2_POST(DI,DX,CX,SI) |
| |
| #define CALC_28 \ |
| CALC_F2_PRE(0xe0,SI,DI,BX) \ |
| PRECALC_4(Y12,0x0) \ |
| CALC_F2_POST(SI,AX,DX,BX) |
| |
| #define CALC_29 \ |
| CALC_F2_PRE(0xe4,BX,SI,CX) \ |
| CALC_F2_POST(BX,DI,AX,CX) |
| |
| #define CALC_30 \ |
| CALC_F2_PRE(0xe8,CX,BX,DX) \ |
| CALC_F2_POST(CX,SI,DI,DX) |
| |
| #define CALC_31 \ |
| CALC_F2_PRE(0xec,DX,CX,AX) \ |
| PRECALC_7(0x30) \ |
| CALC_F2_POST(DX,BX,SI,AX) |
| |
| #define CALC_32 \ |
| CALC_F2_PRE(0x100,AX,DX,DI) \ |
| PRECALC_16(Y15,Y14,Y12,Y8) \ |
| CALC_F2_POST(AX,CX,BX,DI) |
| |
| #define CALC_33 \ |
| CALC_F2_PRE(0x104,DI,AX,SI) \ |
| PRECALC_17(Y15,Y13,Y8) \ |
| CALC_F2_POST(DI,DX,CX,SI) |
| |
| #define CALC_34 \ |
| CALC_F2_PRE(0x108,SI,DI,BX) \ |
| PRECALC_18(Y8) \ |
| CALC_F2_POST(SI,AX,DX,BX) |
| |
| #define CALC_35 \ |
| CALC_F2_PRE(0x10c,BX,SI,CX) \ |
| PRECALC_19(Y8) \ |
| CALC_F2_POST(BX,DI,AX,CX) |
| |
| #define CALC_36 \ |
| CALC_F2_PRE(0x120,CX,BX,DX) \ |
| PRECALC_20(Y8) \ |
| CALC_F2_POST(CX,SI,DI,DX) |
| |
| #define CALC_37 \ |
| CALC_F2_PRE(0x124,DX,CX,AX) \ |
| PRECALC_21(Y8) \ |
| CALC_F2_POST(DX,BX,SI,AX) |
| |
| #define CALC_38 \ |
| CALC_F2_PRE(0x128,AX,DX,DI) \ |
| CALC_F2_POST(AX,CX,BX,DI) |
| |
| |
| #define CALC_F3_PRE(OFFSET,REG_E) \ |
| ADDL OFFSET(R15),REG_E |
| |
| #define CALC_F3_POST(REG_A,REG_B,REG_C,REG_E,REG_TB) \ |
| LEAL (REG_E)(REG_TB*1), REG_E \ // Add F from the previous round |
| MOVL REG_B, BP \ |
| ORL REG_A, BP \ |
| RORXL $0x1b, REG_A, R12 \ |
| RORXL $2, REG_A, REG_TB \ |
| ANDL REG_C, BP \ // Calculate F for the next round |
| ANDL REG_B, REG_A \ |
| ORL BP, REG_A \ |
| ADDL R12, REG_E |
| |
| #define CALC_39 \ |
| CALC_F3_PRE(0x12c,SI) \ |
| PRECALC_23(Y8,0x0,0x80) \ |
| CALC_F3_POST(DI,DX,CX,SI,AX) |
| |
| #define CALC_40 \ |
| CALC_F3_PRE(0x140,BX) \ |
| PRECALC_16(Y14,Y13,Y8,Y7) \ |
| CALC_F3_POST(SI,AX,DX,BX,DI) |
| |
| #define CALC_41 \ |
| CALC_F3_PRE(0x144,CX) \ |
| PRECALC_17(Y14,Y12,Y7) \ |
| CALC_F3_POST(BX,DI,AX,CX,SI) |
| |
| #define CALC_42 \ |
| CALC_F3_PRE(0x148,DX) \ |
| PRECALC_18(Y7) \ |
| CALC_F3_POST(CX,SI,DI,DX,BX) |
| |
| #define CALC_43 \ |
| CALC_F3_PRE(0x14c,AX) \ |
| PRECALC_19(Y7) \ |
| CALC_F3_POST(DX,BX,SI,AX,CX) |
| |
| #define CALC_44 \ |
| CALC_F3_PRE(0x160,DI) \ |
| PRECALC_20(Y7) \ |
| CALC_F3_POST(AX,CX,BX,DI,DX) |
| |
| #define CALC_45 \ |
| CALC_F3_PRE(0x164,SI) \ |
| PRECALC_21(Y7) \ |
| CALC_F3_POST(DI,DX,CX,SI,AX) |
| |
| #define CALC_46 \ |
| CALC_F3_PRE(0x168,BX) \ |
| CALC_F3_POST(SI,AX,DX,BX,DI) |
| |
| #define CALC_47 \ |
| CALC_F3_PRE(0x16c,CX) \ |
| VPXOR Y9, Y0, Y7 \ |
| VPADDD 0x20(R8), Y7, Y0 \ |
| VMOVDQU Y0, 0xa0(R14) \ |
| CALC_F3_POST(BX,DI,AX,CX,SI) |
| |
| #define CALC_48 \ |
| CALC_F3_PRE(0x180,DX) \ |
| PRECALC_16(Y13,Y12,Y7,Y5) \ |
| CALC_F3_POST(CX,SI,DI,DX,BX) |
| |
| #define CALC_49 \ |
| CALC_F3_PRE(0x184,AX) \ |
| PRECALC_17(Y13,Y8,Y5) \ |
| CALC_F3_POST(DX,BX,SI,AX,CX) |
| |
| #define CALC_50 \ |
| CALC_F3_PRE(0x188,DI) \ |
| PRECALC_18(Y5) \ |
| CALC_F3_POST(AX,CX,BX,DI,DX) |
| |
| #define CALC_51 \ |
| CALC_F3_PRE(0x18c,SI) \ |
| PRECALC_19(Y5) \ |
| CALC_F3_POST(DI,DX,CX,SI,AX) |
| |
| #define CALC_52 \ |
| CALC_F3_PRE(0x1a0,BX) \ |
| PRECALC_20(Y5) \ |
| CALC_F3_POST(SI,AX,DX,BX,DI) |
| |
| #define CALC_53 \ |
| CALC_F3_PRE(0x1a4,CX) \ |
| PRECALC_21(Y5) \ |
| CALC_F3_POST(BX,DI,AX,CX,SI) |
| |
| #define CALC_54 \ |
| CALC_F3_PRE(0x1a8,DX) \ |
| CALC_F3_POST(CX,SI,DI,DX,BX) |
| |
| #define CALC_55 \ |
| CALC_F3_PRE(0x1ac,AX) \ |
| PRECALC_23(Y5,0x20,0xc0) \ |
| CALC_F3_POST(DX,BX,SI,AX,CX) |
| |
| #define CALC_56 \ |
| CALC_F3_PRE(0x1c0,DI) \ |
| PRECALC_16(Y12,Y8,Y5,Y3) \ |
| CALC_F3_POST(AX,CX,BX,DI,DX) |
| |
| #define CALC_57 \ |
| CALC_F3_PRE(0x1c4,SI) \ |
| PRECALC_17(Y12,Y7,Y3) \ |
| CALC_F3_POST(DI,DX,CX,SI,AX) |
| |
| #define CALC_58 \ |
| CALC_F3_PRE(0x1c8,BX) \ |
| PRECALC_18(Y3) \ |
| CALC_F3_POST(SI,AX,DX,BX,DI) |
| |
| #define CALC_59 \ |
| CALC_F2_PRE(0x1cc,BX,SI,CX) \ |
| PRECALC_19(Y3) \ |
| CALC_F2_POST(BX,DI,AX,CX) |
| |
| #define CALC_60 \ |
| CALC_F2_PRE(0x1e0,CX,BX,DX) \ |
| PRECALC_20(Y3) \ |
| CALC_F2_POST(CX,SI,DI,DX) |
| |
| #define CALC_61 \ |
| CALC_F2_PRE(0x1e4,DX,CX,AX) \ |
| PRECALC_21(Y3) \ |
| CALC_F2_POST(DX,BX,SI,AX) |
| |
| #define CALC_62 \ |
| CALC_F2_PRE(0x1e8,AX,DX,DI) \ |
| CALC_F2_POST(AX,CX,BX,DI) |
| |
| #define CALC_63 \ |
| CALC_F2_PRE(0x1ec,DI,AX,SI) \ |
| PRECALC_23(Y3,0x20,0xe0) \ |
| CALC_F2_POST(DI,DX,CX,SI) |
| |
| #define CALC_64 \ |
| CALC_F2_PRE(0x200,SI,DI,BX) \ |
| PRECALC_32(Y5,Y3) \ |
| CALC_F2_POST(SI,AX,DX,BX) |
| |
| #define CALC_65 \ |
| CALC_F2_PRE(0x204,BX,SI,CX) \ |
| PRECALC_33(Y14,Y15) \ |
| CALC_F2_POST(BX,DI,AX,CX) |
| |
| #define CALC_66 \ |
| CALC_F2_PRE(0x208,CX,BX,DX) \ |
| PRECALC_34(Y8) \ |
| CALC_F2_POST(CX,SI,DI,DX) |
| |
| #define CALC_67 \ |
| CALC_F2_PRE(0x20c,DX,CX,AX) \ |
| PRECALC_35(Y15) \ |
| CALC_F2_POST(DX,BX,SI,AX) |
| |
| #define CALC_68 \ |
| CALC_F2_PRE(0x220,AX,DX,DI) \ |
| PRECALC_36(Y15) \ |
| CALC_F2_POST(AX,CX,BX,DI) |
| |
| #define CALC_69 \ |
| CALC_F2_PRE(0x224,DI,AX,SI) \ |
| PRECALC_37(Y15) \ |
| CALC_F2_POST(DI,DX,CX,SI) |
| |
| #define CALC_70 \ |
| CALC_F2_PRE(0x228,SI,DI,BX) \ |
| CALC_F2_POST(SI,AX,DX,BX) |
| |
| #define CALC_71 \ |
| CALC_F2_PRE(0x22c,BX,SI,CX) \ |
| PRECALC_39(Y15,0x20,0x100) \ |
| CALC_F2_POST(BX,DI,AX,CX) |
| |
| #define CALC_72 \ |
| CALC_F2_PRE(0x240,CX,BX,DX) \ |
| PRECALC_32(Y3,Y15) \ |
| CALC_F2_POST(CX,SI,DI,DX) |
| |
| #define CALC_73 \ |
| CALC_F2_PRE(0x244,DX,CX,AX) \ |
| PRECALC_33(Y13,Y14) \ |
| CALC_F2_POST(DX,BX,SI,AX) |
| |
| #define CALC_74 \ |
| CALC_F2_PRE(0x248,AX,DX,DI) \ |
| PRECALC_34(Y7) \ |
| CALC_F2_POST(AX,CX,BX,DI) |
| |
| #define CALC_75 \ |
| CALC_F2_PRE(0x24c,DI,AX,SI) \ |
| PRECALC_35(Y14) \ |
| CALC_F2_POST(DI,DX,CX,SI) |
| |
| #define CALC_76 \ |
| CALC_F2_PRE(0x260,SI,DI,BX) \ |
| PRECALC_36(Y14) \ |
| CALC_F2_POST(SI,AX,DX,BX) |
| |
| #define CALC_77 \ |
| CALC_F2_PRE(0x264,BX,SI,CX) \ |
| PRECALC_37(Y14) \ |
| CALC_F2_POST(BX,DI,AX,CX) |
| |
| #define CALC_78 \ |
| CALC_F2_PRE(0x268,CX,BX,DX) \ |
| CALC_F2_POST(CX,SI,DI,DX) |
| |
| #define CALC_79 \ |
| ADDL 0x26c(R15), AX \ |
| LEAL (AX)(CX*1), AX \ |
| RORXL $0x1b, DX, R12 \ |
| PRECALC_39(Y14,0x20,0x120) \ |
| ADDL R12, AX |
| |
| // Similar to CALC_0 |
| #define CALC_80 \ |
| MOVL CX, DX \ |
| RORXL $2, CX, CX \ |
| ANDNL SI, DX, BP \ |
| ANDL BX, DX \ |
| XORL BP, DX \ |
| CALC_F1_PRE(0x10,AX,DX,BX,DI) \ |
| PRECALC_32(Y15,Y14) \ |
| CALC_F1_POST(AX,CX,DI) |
| |
| #define CALC_81 \ |
| CALC_F1_PRE(0x14,DI,AX,CX,SI) \ |
| PRECALC_33(Y12,Y13) \ |
| CALC_F1_POST(DI,DX,SI) |
| |
| #define CALC_82 \ |
| CALC_F1_PRE(0x18,SI,DI,DX,BX) \ |
| PRECALC_34(Y5) \ |
| CALC_F1_POST(SI,AX,BX) |
| |
| #define CALC_83 \ |
| CALC_F1_PRE(0x1c,BX,SI,AX,CX) \ |
| PRECALC_35(Y13) \ |
| CALC_F1_POST(BX,DI,CX) |
| |
| #define CALC_84 \ |
| CALC_F1_PRE(0x30,CX,BX,DI,DX) \ |
| PRECALC_36(Y13) \ |
| CALC_F1_POST(CX,SI,DX) |
| |
| #define CALC_85 \ |
| CALC_F1_PRE(0x34,DX,CX,SI,AX) \ |
| PRECALC_37(Y13) \ |
| CALC_F1_POST(DX,BX,AX) |
| |
| #define CALC_86 \ |
| CALC_F1_PRE(0x38,AX,DX,BX,DI) \ |
| CALC_F1_POST(AX,CX,DI) |
| |
| #define CALC_87 \ |
| CALC_F1_PRE(0x3c,DI,AX,CX,SI) \ |
| PRECALC_39(Y13,0x40,0x140) \ |
| CALC_F1_POST(DI,DX,SI) |
| |
| #define CALC_88 \ |
| CALC_F1_PRE(0x50,SI,DI,DX,BX) \ |
| PRECALC_32(Y14,Y13) \ |
| CALC_F1_POST(SI,AX,BX) |
| |
| #define CALC_89 \ |
| CALC_F1_PRE(0x54,BX,SI,AX,CX) \ |
| PRECALC_33(Y8,Y12) \ |
| CALC_F1_POST(BX,DI,CX) |
| |
| #define CALC_90 \ |
| CALC_F1_PRE(0x58,CX,BX,DI,DX) \ |
| PRECALC_34(Y3) \ |
| CALC_F1_POST(CX,SI,DX) |
| |
| #define CALC_91 \ |
| CALC_F1_PRE(0x5c,DX,CX,SI,AX) \ |
| PRECALC_35(Y12) \ |
| CALC_F1_POST(DX,BX,AX) |
| |
| #define CALC_92 \ |
| CALC_F1_PRE(0x70,AX,DX,BX,DI) \ |
| PRECALC_36(Y12) \ |
| CALC_F1_POST(AX,CX,DI) |
| |
| #define CALC_93 \ |
| CALC_F1_PRE(0x74,DI,AX,CX,SI) \ |
| PRECALC_37(Y12) \ |
| CALC_F1_POST(DI,DX,SI) |
| |
| #define CALC_94 \ |
| CALC_F1_PRE(0x78,SI,DI,DX,BX) \ |
| CALC_F1_POST(SI,AX,BX) |
| |
| #define CALC_95 \ |
| CALC_F1_PRE(0x7c,BX,SI,AX,CX) \ |
| PRECALC_39(Y12,0x40,0x160) \ |
| CALC_F1_POST(BX,DI,CX) |
| |
| #define CALC_96 \ |
| CALC_F1_PRE(0x90,CX,BX,DI,DX) \ |
| PRECALC_32(Y13,Y12) \ |
| CALC_F1_POST(CX,SI,DX) |
| |
| #define CALC_97 \ |
| CALC_F1_PRE(0x94,DX,CX,SI,AX) \ |
| PRECALC_33(Y7,Y8) \ |
| CALC_F1_POST(DX,BX,AX) |
| |
| #define CALC_98 \ |
| CALC_F1_PRE(0x98,AX,DX,BX,DI) \ |
| PRECALC_34(Y15) \ |
| CALC_F1_POST(AX,CX,DI) |
| |
| #define CALC_99 \ |
| CALC_F2_PRE(0x9c,DI,AX,SI) \ |
| PRECALC_35(Y8) \ |
| CALC_F2_POST(DI,DX,CX,SI) |
| |
| #define CALC_100 \ |
| CALC_F2_PRE(0xb0,SI,DI,BX) \ |
| PRECALC_36(Y8) \ |
| CALC_F2_POST(SI,AX,DX,BX) |
| |
| #define CALC_101 \ |
| CALC_F2_PRE(0xb4,BX,SI,CX) \ |
| PRECALC_37(Y8) \ |
| CALC_F2_POST(BX,DI,AX,CX) |
| |
| #define CALC_102 \ |
| CALC_F2_PRE(0xb8,CX,BX,DX) \ |
| CALC_F2_POST(CX,SI,DI,DX) |
| |
| #define CALC_103 \ |
| CALC_F2_PRE(0xbc,DX,CX,AX) \ |
| PRECALC_39(Y8,0x40,0x180) \ |
| CALC_F2_POST(DX,BX,SI,AX) |
| |
| #define CALC_104 \ |
| CALC_F2_PRE(0xd0,AX,DX,DI) \ |
| PRECALC_32(Y12,Y8) \ |
| CALC_F2_POST(AX,CX,BX,DI) |
| |
| #define CALC_105 \ |
| CALC_F2_PRE(0xd4,DI,AX,SI) \ |
| PRECALC_33(Y5,Y7) \ |
| CALC_F2_POST(DI,DX,CX,SI) |
| |
| #define CALC_106 \ |
| CALC_F2_PRE(0xd8,SI,DI,BX) \ |
| PRECALC_34(Y14) \ |
| CALC_F2_POST(SI,AX,DX,BX) |
| |
| #define CALC_107 \ |
| CALC_F2_PRE(0xdc,BX,SI,CX) \ |
| PRECALC_35(Y7) \ |
| CALC_F2_POST(BX,DI,AX,CX) |
| |
| #define CALC_108 \ |
| CALC_F2_PRE(0xf0,CX,BX,DX) \ |
| PRECALC_36(Y7) \ |
| CALC_F2_POST(CX,SI,DI,DX) |
| |
| #define CALC_109 \ |
| CALC_F2_PRE(0xf4,DX,CX,AX) \ |
| PRECALC_37(Y7) \ |
| CALC_F2_POST(DX,BX,SI,AX) |
| |
| #define CALC_110 \ |
| CALC_F2_PRE(0xf8,AX,DX,DI) \ |
| CALC_F2_POST(AX,CX,BX,DI) |
| |
| #define CALC_111 \ |
| CALC_F2_PRE(0xfc,DI,AX,SI) \ |
| PRECALC_39(Y7,0x40,0x1a0) \ |
| CALC_F2_POST(DI,DX,CX,SI) |
| |
| #define CALC_112 \ |
| CALC_F2_PRE(0x110,SI,DI,BX) \ |
| PRECALC_32(Y8,Y7) \ |
| CALC_F2_POST(SI,AX,DX,BX) |
| |
| #define CALC_113 \ |
| CALC_F2_PRE(0x114,BX,SI,CX) \ |
| PRECALC_33(Y3,Y5) \ |
| CALC_F2_POST(BX,DI,AX,CX) |
| |
| #define CALC_114 \ |
| CALC_F2_PRE(0x118,CX,BX,DX) \ |
| PRECALC_34(Y13) \ |
| CALC_F2_POST(CX,SI,DI,DX) |
| |
| #define CALC_115 \ |
| CALC_F2_PRE(0x11c,DX,CX,AX) \ |
| PRECALC_35(Y5) \ |
| CALC_F2_POST(DX,BX,SI,AX) |
| |
| #define CALC_116 \ |
| CALC_F2_PRE(0x130,AX,DX,DI) \ |
| PRECALC_36(Y5) \ |
| CALC_F2_POST(AX,CX,BX,DI) |
| |
| #define CALC_117 \ |
| CALC_F2_PRE(0x134,DI,AX,SI) \ |
| PRECALC_37(Y5) \ |
| CALC_F2_POST(DI,DX,CX,SI) |
| |
| #define CALC_118 \ |
| CALC_F2_PRE(0x138,SI,DI,BX) \ |
| CALC_F2_POST(SI,AX,DX,BX) |
| |
| #define CALC_119 \ |
| CALC_F3_PRE(0x13c,CX) \ |
| PRECALC_39(Y5,0x40,0x1c0) \ |
| CALC_F3_POST(BX,DI,AX,CX,SI) |
| |
| #define CALC_120 \ |
| CALC_F3_PRE(0x150,DX) \ |
| PRECALC_32(Y7,Y5) \ |
| CALC_F3_POST(CX,SI,DI,DX,BX) |
| |
| #define CALC_121 \ |
| CALC_F3_PRE(0x154,AX) \ |
| PRECALC_33(Y15,Y3) \ |
| CALC_F3_POST(DX,BX,SI,AX,CX) |
| |
| #define CALC_122 \ |
| CALC_F3_PRE(0x158,DI) \ |
| PRECALC_34(Y12) \ |
| CALC_F3_POST(AX,CX,BX,DI,DX) |
| |
| #define CALC_123 \ |
| CALC_F3_PRE(0x15c,SI) \ |
| PRECALC_35(Y3) \ |
| CALC_F3_POST(DI,DX,CX,SI,AX) |
| |
| #define CALC_124 \ |
| CALC_F3_PRE(0x170,BX) \ |
| PRECALC_36(Y3) \ |
| CALC_F3_POST(SI,AX,DX,BX,DI) |
| |
| #define CALC_125 \ |
| CALC_F3_PRE(0x174,CX) \ |
| PRECALC_37(Y3) \ |
| CALC_F3_POST(BX,DI,AX,CX,SI) |
| |
| #define CALC_126 \ |
| CALC_F3_PRE(0x178,DX) \ |
| CALC_F3_POST(CX,SI,DI,DX,BX) |
| |
| #define CALC_127 \ |
| CALC_F3_PRE(0x17c,AX) \ |
| PRECALC_39(Y3,0x60,0x1e0) \ |
| CALC_F3_POST(DX,BX,SI,AX,CX) |
| |
| #define CALC_128 \ |
| CALC_F3_PRE(0x190,DI) \ |
| PRECALC_32(Y5,Y3) \ |
| CALC_F3_POST(AX,CX,BX,DI,DX) |
| |
| #define CALC_129 \ |
| CALC_F3_PRE(0x194,SI) \ |
| PRECALC_33(Y14,Y15) \ |
| CALC_F3_POST(DI,DX,CX,SI,AX) |
| |
| #define CALC_130 \ |
| CALC_F3_PRE(0x198,BX) \ |
| PRECALC_34(Y8) \ |
| CALC_F3_POST(SI,AX,DX,BX,DI) |
| |
| #define CALC_131 \ |
| CALC_F3_PRE(0x19c,CX) \ |
| PRECALC_35(Y15) \ |
| CALC_F3_POST(BX,DI,AX,CX,SI) |
| |
| #define CALC_132 \ |
| CALC_F3_PRE(0x1b0,DX) \ |
| PRECALC_36(Y15) \ |
| CALC_F3_POST(CX,SI,DI,DX,BX) |
| |
| #define CALC_133 \ |
| CALC_F3_PRE(0x1b4,AX) \ |
| PRECALC_37(Y15) \ |
| CALC_F3_POST(DX,BX,SI,AX,CX) |
| |
| #define CALC_134 \ |
| CALC_F3_PRE(0x1b8,DI) \ |
| CALC_F3_POST(AX,CX,BX,DI,DX) |
| |
| #define CALC_135 \ |
| CALC_F3_PRE(0x1bc,SI) \ |
| PRECALC_39(Y15,0x60,0x200) \ |
| CALC_F3_POST(DI,DX,CX,SI,AX) |
| |
| #define CALC_136 \ |
| CALC_F3_PRE(0x1d0,BX) \ |
| PRECALC_32(Y3,Y15) \ |
| CALC_F3_POST(SI,AX,DX,BX,DI) |
| |
| #define CALC_137 \ |
| CALC_F3_PRE(0x1d4,CX) \ |
| PRECALC_33(Y13,Y14) \ |
| CALC_F3_POST(BX,DI,AX,CX,SI) |
| |
| #define CALC_138 \ |
| CALC_F3_PRE(0x1d8,DX) \ |
| PRECALC_34(Y7) \ |
| CALC_F3_POST(CX,SI,DI,DX,BX) |
| |
| #define CALC_139 \ |
| CALC_F2_PRE(0x1dc,DX,CX,AX) \ |
| PRECALC_35(Y14) \ |
| CALC_F2_POST(DX,BX,SI,AX) |
| |
| #define CALC_140 \ |
| CALC_F2_PRE(0x1f0,AX,DX,DI) \ |
| PRECALC_36(Y14) \ |
| CALC_F2_POST(AX,CX,BX,DI) |
| |
| #define CALC_141 \ |
| CALC_F2_PRE(0x1f4,DI,AX,SI) \ |
| PRECALC_37(Y14) \ |
| CALC_F2_POST(DI,DX,CX,SI) |
| |
| #define CALC_142 \ |
| CALC_F2_PRE(0x1f8,SI,DI,BX) \ |
| CALC_F2_POST(SI,AX,DX,BX) |
| |
| #define CALC_143 \ |
| CALC_F2_PRE(0x1fc,BX,SI,CX) \ |
| PRECALC_39(Y14,0x60,0x220) \ |
| CALC_F2_POST(BX,DI,AX,CX) |
| |
| #define CALC_144 \ |
| CALC_F2_PRE(0x210,CX,BX,DX) \ |
| PRECALC_32(Y15,Y14) \ |
| CALC_F2_POST(CX,SI,DI,DX) |
| |
| #define CALC_145 \ |
| CALC_F2_PRE(0x214,DX,CX,AX) \ |
| PRECALC_33(Y12,Y13) \ |
| CALC_F2_POST(DX,BX,SI,AX) |
| |
| #define CALC_146 \ |
| CALC_F2_PRE(0x218,AX,DX,DI) \ |
| PRECALC_34(Y5) \ |
| CALC_F2_POST(AX,CX,BX,DI) |
| |
| #define CALC_147 \ |
| CALC_F2_PRE(0x21c,DI,AX,SI) \ |
| PRECALC_35(Y13) \ |
| CALC_F2_POST(DI,DX,CX,SI) |
| |
| #define CALC_148 \ |
| CALC_F2_PRE(0x230,SI,DI,BX) \ |
| PRECALC_36(Y13) \ |
| CALC_F2_POST(SI,AX,DX,BX) |
| |
| #define CALC_149 \ |
| CALC_F2_PRE(0x234,BX,SI,CX) \ |
| PRECALC_37(Y13) \ |
| CALC_F2_POST(BX,DI,AX,CX) |
| |
| #define CALC_150 \ |
| CALC_F2_PRE(0x238,CX,BX,DX) \ |
| CALC_F2_POST(CX,SI,DI,DX) |
| |
| #define CALC_151 \ |
| CALC_F2_PRE(0x23c,DX,CX,AX) \ |
| PRECALC_39(Y13,0x60,0x240) \ |
| CALC_F2_POST(DX,BX,SI,AX) |
| |
| #define CALC_152 \ |
| CALC_F2_PRE(0x250,AX,DX,DI) \ |
| PRECALC_32(Y14,Y13) \ |
| CALC_F2_POST(AX,CX,BX,DI) |
| |
| #define CALC_153 \ |
| CALC_F2_PRE(0x254,DI,AX,SI) \ |
| PRECALC_33(Y8,Y12) \ |
| CALC_F2_POST(DI,DX,CX,SI) |
| |
| #define CALC_154 \ |
| CALC_F2_PRE(0x258,SI,DI,BX) \ |
| PRECALC_34(Y3) \ |
| CALC_F2_POST(SI,AX,DX,BX) |
| |
| #define CALC_155 \ |
| CALC_F2_PRE(0x25c,BX,SI,CX) \ |
| PRECALC_35(Y12) \ |
| CALC_F2_POST(BX,DI,AX,CX) |
| |
| #define CALC_156 \ |
| CALC_F2_PRE(0x270,CX,BX,DX) \ |
| PRECALC_36(Y12) \ |
| CALC_F2_POST(CX,SI,DI,DX) |
| |
| #define CALC_157 \ |
| CALC_F2_PRE(0x274,DX,CX,AX) \ |
| PRECALC_37(Y12) \ |
| CALC_F2_POST(DX,BX,SI,AX) |
| |
| #define CALC_158 \ |
| CALC_F2_PRE(0x278,AX,DX,DI) \ |
| CALC_F2_POST(AX,CX,BX,DI) |
| |
| #define CALC_159 \ |
| ADDL 0x27c(R15),SI \ |
| LEAL (SI)(AX*1), SI \ |
| RORXL $0x1b, DI, R12 \ |
| PRECALC_39(Y12,0x60,0x260) \ |
| ADDL R12, SI |
| |
| |
| |
| #define CALC \ |
| MOVL (R9), CX \ |
| MOVL 4(R9), SI \ |
| MOVL 8(R9), DI \ |
| MOVL 12(R9), AX \ |
| MOVL 16(R9), DX \ |
| MOVQ SP, R14 \ |
| LEAQ (2*4*80+32)(SP), R15 \ |
| PRECALC \ // Precalc WK for first 2 blocks |
| XCHGQ R15, R14 \ |
| loop: \ // this loops is unrolled |
| CMPQ R10, R8 \ // we use R8 value (set below) as a signal of a last block |
| JNE begin \ |
| VZEROUPPER \ |
| RET \ |
| begin: \ |
| CALC_0 \ |
| CALC_1 \ |
| CALC_2 \ |
| CALC_3 \ |
| CALC_4 \ |
| CALC_5 \ |
| CALC_6 \ |
| CALC_7 \ |
| CALC_8 \ |
| CALC_9 \ |
| CALC_10 \ |
| CALC_11 \ |
| CALC_12 \ |
| CALC_13 \ |
| CALC_14 \ |
| CALC_15 \ |
| CALC_16 \ |
| CALC_17 \ |
| CALC_18 \ |
| CALC_19 \ |
| CALC_20 \ |
| CALC_21 \ |
| CALC_22 \ |
| CALC_23 \ |
| CALC_24 \ |
| CALC_25 \ |
| CALC_26 \ |
| CALC_27 \ |
| CALC_28 \ |
| CALC_29 \ |
| CALC_30 \ |
| CALC_31 \ |
| CALC_32 \ |
| CALC_33 \ |
| CALC_34 \ |
| CALC_35 \ |
| CALC_36 \ |
| CALC_37 \ |
| CALC_38 \ |
| CALC_39 \ |
| CALC_40 \ |
| CALC_41 \ |
| CALC_42 \ |
| CALC_43 \ |
| CALC_44 \ |
| CALC_45 \ |
| CALC_46 \ |
| CALC_47 \ |
| CALC_48 \ |
| CALC_49 \ |
| CALC_50 \ |
| CALC_51 \ |
| CALC_52 \ |
| CALC_53 \ |
| CALC_54 \ |
| CALC_55 \ |
| CALC_56 \ |
| CALC_57 \ |
| CALC_58 \ |
| CALC_59 \ |
| ADDQ $128, R10 \ // move to next even-64-byte block |
| CMPQ R10, R11 \ // is current block the last one? |
| CMOVQCC R8, R10 \ // signal the last iteration smartly |
| CALC_60 \ |
| CALC_61 \ |
| CALC_62 \ |
| CALC_63 \ |
| CALC_64 \ |
| CALC_65 \ |
| CALC_66 \ |
| CALC_67 \ |
| CALC_68 \ |
| CALC_69 \ |
| CALC_70 \ |
| CALC_71 \ |
| CALC_72 \ |
| CALC_73 \ |
| CALC_74 \ |
| CALC_75 \ |
| CALC_76 \ |
| CALC_77 \ |
| CALC_78 \ |
| CALC_79 \ |
| UPDATE_HASH(AX,DX,BX,SI,DI) \ |
| CMPQ R10, R8 \ // is current block the last one? |
| JE loop\ |
| MOVL DX, CX \ |
| CALC_80 \ |
| CALC_81 \ |
| CALC_82 \ |
| CALC_83 \ |
| CALC_84 \ |
| CALC_85 \ |
| CALC_86 \ |
| CALC_87 \ |
| CALC_88 \ |
| CALC_89 \ |
| CALC_90 \ |
| CALC_91 \ |
| CALC_92 \ |
| CALC_93 \ |
| CALC_94 \ |
| CALC_95 \ |
| CALC_96 \ |
| CALC_97 \ |
| CALC_98 \ |
| CALC_99 \ |
| CALC_100 \ |
| CALC_101 \ |
| CALC_102 \ |
| CALC_103 \ |
| CALC_104 \ |
| CALC_105 \ |
| CALC_106 \ |
| CALC_107 \ |
| CALC_108 \ |
| CALC_109 \ |
| CALC_110 \ |
| CALC_111 \ |
| CALC_112 \ |
| CALC_113 \ |
| CALC_114 \ |
| CALC_115 \ |
| CALC_116 \ |
| CALC_117 \ |
| CALC_118 \ |
| CALC_119 \ |
| CALC_120 \ |
| CALC_121 \ |
| CALC_122 \ |
| CALC_123 \ |
| CALC_124 \ |
| CALC_125 \ |
| CALC_126 \ |
| CALC_127 \ |
| CALC_128 \ |
| CALC_129 \ |
| CALC_130 \ |
| CALC_131 \ |
| CALC_132 \ |
| CALC_133 \ |
| CALC_134 \ |
| CALC_135 \ |
| CALC_136 \ |
| CALC_137 \ |
| CALC_138 \ |
| CALC_139 \ |
| ADDQ $128, R13 \ //move to next even-64-byte block |
| CMPQ R13, R11 \ //is current block the last one? |
| CMOVQCC R8, R10 \ |
| CALC_140 \ |
| CALC_141 \ |
| CALC_142 \ |
| CALC_143 \ |
| CALC_144 \ |
| CALC_145 \ |
| CALC_146 \ |
| CALC_147 \ |
| CALC_148 \ |
| CALC_149 \ |
| CALC_150 \ |
| CALC_151 \ |
| CALC_152 \ |
| CALC_153 \ |
| CALC_154 \ |
| CALC_155 \ |
| CALC_156 \ |
| CALC_157 \ |
| CALC_158 \ |
| CALC_159 \ |
| UPDATE_HASH(SI,DI,DX,CX,BX) \ |
| MOVL SI, R12 \ //Reset state for AVX2 reg permutation |
| MOVL DI, SI \ |
| MOVL DX, DI \ |
| MOVL BX, DX \ |
| MOVL CX, AX \ |
| MOVL R12, CX \ |
| XCHGQ R15, R14 \ |
| JMP loop |
| |
| |
| |
| TEXT ·blockAVX2(SB),$1408-32 |
| |
| MOVQ dig+0(FP), DI |
| MOVQ p_base+8(FP), SI |
| MOVQ p_len+16(FP), DX |
| SHRQ $6, DX |
| SHLQ $6, DX |
| |
| MOVQ $K_XMM_AR<>(SB), R8 |
| |
| MOVQ DI, R9 |
| MOVQ SI, R10 |
| LEAQ 64(SI), R13 |
| |
| ADDQ SI, DX |
| ADDQ $64, DX |
| MOVQ DX, R11 |
| |
| CMPQ R13, R11 |
| CMOVQCC R8, R13 |
| |
| VMOVDQU BSWAP_SHUFB_CTL<>(SB), Y10 |
| |
| CALC // RET is inside macros |
| |
| DATA K_XMM_AR<>+0x00(SB)/4,$0x5a827999 |
| DATA K_XMM_AR<>+0x04(SB)/4,$0x5a827999 |
| DATA K_XMM_AR<>+0x08(SB)/4,$0x5a827999 |
| DATA K_XMM_AR<>+0x0c(SB)/4,$0x5a827999 |
| DATA K_XMM_AR<>+0x10(SB)/4,$0x5a827999 |
| DATA K_XMM_AR<>+0x14(SB)/4,$0x5a827999 |
| DATA K_XMM_AR<>+0x18(SB)/4,$0x5a827999 |
| DATA K_XMM_AR<>+0x1c(SB)/4,$0x5a827999 |
| DATA K_XMM_AR<>+0x20(SB)/4,$0x6ed9eba1 |
| DATA K_XMM_AR<>+0x24(SB)/4,$0x6ed9eba1 |
| DATA K_XMM_AR<>+0x28(SB)/4,$0x6ed9eba1 |
| DATA K_XMM_AR<>+0x2c(SB)/4,$0x6ed9eba1 |
| DATA K_XMM_AR<>+0x30(SB)/4,$0x6ed9eba1 |
| DATA K_XMM_AR<>+0x34(SB)/4,$0x6ed9eba1 |
| DATA K_XMM_AR<>+0x38(SB)/4,$0x6ed9eba1 |
| DATA K_XMM_AR<>+0x3c(SB)/4,$0x6ed9eba1 |
| DATA K_XMM_AR<>+0x40(SB)/4,$0x8f1bbcdc |
| DATA K_XMM_AR<>+0x44(SB)/4,$0x8f1bbcdc |
| DATA K_XMM_AR<>+0x48(SB)/4,$0x8f1bbcdc |
| DATA K_XMM_AR<>+0x4c(SB)/4,$0x8f1bbcdc |
| DATA K_XMM_AR<>+0x50(SB)/4,$0x8f1bbcdc |
| DATA K_XMM_AR<>+0x54(SB)/4,$0x8f1bbcdc |
| DATA K_XMM_AR<>+0x58(SB)/4,$0x8f1bbcdc |
| DATA K_XMM_AR<>+0x5c(SB)/4,$0x8f1bbcdc |
| DATA K_XMM_AR<>+0x60(SB)/4,$0xca62c1d6 |
| DATA K_XMM_AR<>+0x64(SB)/4,$0xca62c1d6 |
| DATA K_XMM_AR<>+0x68(SB)/4,$0xca62c1d6 |
| DATA K_XMM_AR<>+0x6c(SB)/4,$0xca62c1d6 |
| DATA K_XMM_AR<>+0x70(SB)/4,$0xca62c1d6 |
| DATA K_XMM_AR<>+0x74(SB)/4,$0xca62c1d6 |
| DATA K_XMM_AR<>+0x78(SB)/4,$0xca62c1d6 |
| DATA K_XMM_AR<>+0x7c(SB)/4,$0xca62c1d6 |
| GLOBL K_XMM_AR<>(SB),RODATA,$128 |
| |
| DATA BSWAP_SHUFB_CTL<>+0x00(SB)/4,$0x00010203 |
| DATA BSWAP_SHUFB_CTL<>+0x04(SB)/4,$0x04050607 |
| DATA BSWAP_SHUFB_CTL<>+0x08(SB)/4,$0x08090a0b |
| DATA BSWAP_SHUFB_CTL<>+0x0c(SB)/4,$0x0c0d0e0f |
| DATA BSWAP_SHUFB_CTL<>+0x10(SB)/4,$0x00010203 |
| DATA BSWAP_SHUFB_CTL<>+0x14(SB)/4,$0x04050607 |
| DATA BSWAP_SHUFB_CTL<>+0x18(SB)/4,$0x08090a0b |
| DATA BSWAP_SHUFB_CTL<>+0x1c(SB)/4,$0x0c0d0e0f |
| GLOBL BSWAP_SHUFB_CTL<>(SB),RODATA,$32 |
