| // Copyright 2015 The Go Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style | |
| // license that can be found in the LICENSE file. | |
| // This is an optimized implementation of AES-GCM using AES-NI and CLMUL-NI | |
| // The implementation uses some optimization as described in: | |
| // [1] Gueron, S., Kounavis, M.E.: Intel® Carry-Less Multiplication | |
| // Instruction and its Usage for Computing the GCM Mode rev. 2.02 | |
| // [2] Gueron, S., Krasnov, V.: Speeding up Counter Mode in Software and | |
| // Hardware | |
| #include "textflag.h" | |
| #define B0 X0 | |
| #define B1 X1 | |
| #define B2 X2 | |
| #define B3 X3 | |
| #define B4 X4 | |
| #define B5 X5 | |
| #define B6 X6 | |
| #define B7 X7 | |
| #define ACC0 X8 | |
| #define ACC1 X9 | |
| #define ACCM X10 | |
| #define T0 X11 | |
| #define T1 X12 | |
| #define T2 X13 | |
| #define POLY X14 | |
| #define BSWAP X15 | |
| DATA bswapMask<>+0x00(SB)/8, $0x08090a0b0c0d0e0f | |
| DATA bswapMask<>+0x08(SB)/8, $0x0001020304050607 | |
| DATA gcmPoly<>+0x00(SB)/8, $0x0000000000000001 | |
| DATA gcmPoly<>+0x08(SB)/8, $0xc200000000000000 | |
| DATA andMask<>+0x00(SB)/8, $0x00000000000000ff | |
| DATA andMask<>+0x08(SB)/8, $0x0000000000000000 | |
| DATA andMask<>+0x10(SB)/8, $0x000000000000ffff | |
| DATA andMask<>+0x18(SB)/8, $0x0000000000000000 | |
| DATA andMask<>+0x20(SB)/8, $0x0000000000ffffff | |
| DATA andMask<>+0x28(SB)/8, $0x0000000000000000 | |
| DATA andMask<>+0x30(SB)/8, $0x00000000ffffffff | |
| DATA andMask<>+0x38(SB)/8, $0x0000000000000000 | |
| DATA andMask<>+0x40(SB)/8, $0x000000ffffffffff | |
| DATA andMask<>+0x48(SB)/8, $0x0000000000000000 | |
| DATA andMask<>+0x50(SB)/8, $0x0000ffffffffffff | |
| DATA andMask<>+0x58(SB)/8, $0x0000000000000000 | |
| DATA andMask<>+0x60(SB)/8, $0x00ffffffffffffff | |
| DATA andMask<>+0x68(SB)/8, $0x0000000000000000 | |
| DATA andMask<>+0x70(SB)/8, $0xffffffffffffffff | |
| DATA andMask<>+0x78(SB)/8, $0x0000000000000000 | |
| DATA andMask<>+0x80(SB)/8, $0xffffffffffffffff | |
| DATA andMask<>+0x88(SB)/8, $0x00000000000000ff | |
| DATA andMask<>+0x90(SB)/8, $0xffffffffffffffff | |
| DATA andMask<>+0x98(SB)/8, $0x000000000000ffff | |
| DATA andMask<>+0xa0(SB)/8, $0xffffffffffffffff | |
| DATA andMask<>+0xa8(SB)/8, $0x0000000000ffffff | |
| DATA andMask<>+0xb0(SB)/8, $0xffffffffffffffff | |
| DATA andMask<>+0xb8(SB)/8, $0x00000000ffffffff | |
| DATA andMask<>+0xc0(SB)/8, $0xffffffffffffffff | |
| DATA andMask<>+0xc8(SB)/8, $0x000000ffffffffff | |
| DATA andMask<>+0xd0(SB)/8, $0xffffffffffffffff | |
| DATA andMask<>+0xd8(SB)/8, $0x0000ffffffffffff | |
| DATA andMask<>+0xe0(SB)/8, $0xffffffffffffffff | |
| DATA andMask<>+0xe8(SB)/8, $0x00ffffffffffffff | |
| GLOBL bswapMask<>(SB), (NOPTR+RODATA), $16 | |
| GLOBL gcmPoly<>(SB), (NOPTR+RODATA), $16 | |
| GLOBL andMask<>(SB), (NOPTR+RODATA), $240 | |
| // func gcmAesFinish(productTable *[256]byte, tagMask, T *[16]byte, pLen, dLen uint64) | |
| TEXT ·gcmAesFinish(SB),NOSPLIT,$0 | |
| #define pTbl DI | |
| #define tMsk SI | |
| #define tPtr DX | |
| #define plen AX | |
| #define dlen CX | |
| MOVQ productTable+0(FP), pTbl | |
| MOVQ tagMask+8(FP), tMsk | |
| MOVQ T+16(FP), tPtr | |
| MOVQ pLen+24(FP), plen | |
| MOVQ dLen+32(FP), dlen | |
| MOVOU (tPtr), ACC0 | |
| MOVOU (tMsk), T2 | |
| MOVOU bswapMask<>(SB), BSWAP | |
| MOVOU gcmPoly<>(SB), POLY | |
| SHLQ $3, plen | |
| SHLQ $3, dlen | |
| MOVQ plen, B0 | |
| PINSRQ $1, dlen, B0 | |
| PXOR ACC0, B0 | |
| MOVOU (16*14)(pTbl), ACC0 | |
| MOVOU (16*15)(pTbl), ACCM | |
| MOVOU ACC0, ACC1 | |
| PCLMULQDQ $0x00, B0, ACC0 | |
| PCLMULQDQ $0x11, B0, ACC1 | |
| PSHUFD $78, B0, T0 | |
| PXOR B0, T0 | |
| PCLMULQDQ $0x00, T0, ACCM | |
| PXOR ACC0, ACCM | |
| PXOR ACC1, ACCM | |
| MOVOU ACCM, T0 | |
| PSRLDQ $8, ACCM | |
| PSLLDQ $8, T0 | |
| PXOR ACCM, ACC1 | |
| PXOR T0, ACC0 | |
| MOVOU POLY, T0 | |
| PCLMULQDQ $0x01, ACC0, T0 | |
| PSHUFD $78, ACC0, ACC0 | |
| PXOR T0, ACC0 | |
| MOVOU POLY, T0 | |
| PCLMULQDQ $0x01, ACC0, T0 | |
| PSHUFD $78, ACC0, ACC0 | |
| PXOR T0, ACC0 | |
| PXOR ACC1, ACC0 | |
| PSHUFB BSWAP, ACC0 | |
| PXOR T2, ACC0 | |
| MOVOU ACC0, (tPtr) | |
| RET | |
| #undef pTbl | |
| #undef tMsk | |
| #undef tPtr | |
| #undef plen | |
| #undef dlen | |
| // func gcmAesInit(productTable *[256]byte, ks []uint32) | |
| TEXT ·gcmAesInit(SB),NOSPLIT,$0 | |
| #define dst DI | |
| #define KS SI | |
| #define NR DX | |
| MOVQ productTable+0(FP), dst | |
| MOVQ ks_base+8(FP), KS | |
| MOVQ ks_len+16(FP), NR | |
| SHRQ $2, NR | |
| DECQ NR | |
| MOVOU bswapMask<>(SB), BSWAP | |
| MOVOU gcmPoly<>(SB), POLY | |
| // Encrypt block 0, with the AES key to generate the hash key H | |
| MOVOU (16*0)(KS), B0 | |
| MOVOU (16*1)(KS), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*2)(KS), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*3)(KS), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*4)(KS), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*5)(KS), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*6)(KS), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*7)(KS), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*8)(KS), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*9)(KS), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*10)(KS), T0 | |
| CMPQ NR, $12 | |
| JB initEncLast | |
| AESENC T0, B0 | |
| MOVOU (16*11)(KS), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*12)(KS), T0 | |
| JE initEncLast | |
| AESENC T0, B0 | |
| MOVOU (16*13)(KS), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*14)(KS), T0 | |
| initEncLast: | |
| AESENCLAST T0, B0 | |
| PSHUFB BSWAP, B0 | |
| // H * 2 | |
| PSHUFD $0xff, B0, T0 | |
| MOVOU B0, T1 | |
| PSRAL $31, T0 | |
| PAND POLY, T0 | |
| PSRLL $31, T1 | |
| PSLLDQ $4, T1 | |
| PSLLL $1, B0 | |
| PXOR T0, B0 | |
| PXOR T1, B0 | |
| // Karatsuba pre-computations | |
| MOVOU B0, (16*14)(dst) | |
| PSHUFD $78, B0, B1 | |
| PXOR B0, B1 | |
| MOVOU B1, (16*15)(dst) | |
| MOVOU B0, B2 | |
| MOVOU B1, B3 | |
| // Now prepare powers of H and pre-computations for them | |
| MOVQ $7, AX | |
| initLoop: | |
| MOVOU B2, T0 | |
| MOVOU B2, T1 | |
| MOVOU B3, T2 | |
| PCLMULQDQ $0x00, B0, T0 | |
| PCLMULQDQ $0x11, B0, T1 | |
| PCLMULQDQ $0x00, B1, T2 | |
| PXOR T0, T2 | |
| PXOR T1, T2 | |
| MOVOU T2, B4 | |
| PSLLDQ $8, B4 | |
| PSRLDQ $8, T2 | |
| PXOR B4, T0 | |
| PXOR T2, T1 | |
| MOVOU POLY, B2 | |
| PCLMULQDQ $0x01, T0, B2 | |
| PSHUFD $78, T0, T0 | |
| PXOR B2, T0 | |
| MOVOU POLY, B2 | |
| PCLMULQDQ $0x01, T0, B2 | |
| PSHUFD $78, T0, T0 | |
| PXOR T0, B2 | |
| PXOR T1, B2 | |
| MOVOU B2, (16*12)(dst) | |
| PSHUFD $78, B2, B3 | |
| PXOR B2, B3 | |
| MOVOU B3, (16*13)(dst) | |
| DECQ AX | |
| LEAQ (-16*2)(dst), dst | |
| JNE initLoop | |
| RET | |
| #undef NR | |
| #undef KS | |
| #undef dst | |
| // func gcmAesData(productTable *[256]byte, data []byte, T *[16]byte) | |
| TEXT ·gcmAesData(SB),NOSPLIT,$0 | |
| #define pTbl DI | |
| #define aut SI | |
| #define tPtr CX | |
| #define autLen DX | |
| #define reduceRound(a) MOVOU POLY, T0; PCLMULQDQ $0x01, a, T0; PSHUFD $78, a, a; PXOR T0, a | |
| #define mulRoundAAD(X ,i) \ | |
| MOVOU (16*(i*2))(pTbl), T1;\ | |
| MOVOU T1, T2;\ | |
| PCLMULQDQ $0x00, X, T1;\ | |
| PXOR T1, ACC0;\ | |
| PCLMULQDQ $0x11, X, T2;\ | |
| PXOR T2, ACC1;\ | |
| PSHUFD $78, X, T1;\ | |
| PXOR T1, X;\ | |
| MOVOU (16*(i*2+1))(pTbl), T1;\ | |
| PCLMULQDQ $0x00, X, T1;\ | |
| PXOR T1, ACCM | |
| MOVQ productTable+0(FP), pTbl | |
| MOVQ data_base+8(FP), aut | |
| MOVQ data_len+16(FP), autLen | |
| MOVQ T+32(FP), tPtr | |
| PXOR ACC0, ACC0 | |
| MOVOU bswapMask<>(SB), BSWAP | |
| MOVOU gcmPoly<>(SB), POLY | |
| TESTQ autLen, autLen | |
| JEQ dataBail | |
| CMPQ autLen, $13 // optimize the TLS case | |
| JE dataTLS | |
| CMPQ autLen, $128 | |
| JB startSinglesLoop | |
| JMP dataOctaLoop | |
| dataTLS: | |
| MOVOU (16*14)(pTbl), T1 | |
| MOVOU (16*15)(pTbl), T2 | |
| PXOR B0, B0 | |
| MOVQ (aut), B0 | |
| PINSRD $2, 8(aut), B0 | |
| PINSRB $12, 12(aut), B0 | |
| XORQ autLen, autLen | |
| JMP dataMul | |
| dataOctaLoop: | |
| CMPQ autLen, $128 | |
| JB startSinglesLoop | |
| SUBQ $128, autLen | |
| MOVOU (16*0)(aut), X0 | |
| MOVOU (16*1)(aut), X1 | |
| MOVOU (16*2)(aut), X2 | |
| MOVOU (16*3)(aut), X3 | |
| MOVOU (16*4)(aut), X4 | |
| MOVOU (16*5)(aut), X5 | |
| MOVOU (16*6)(aut), X6 | |
| MOVOU (16*7)(aut), X7 | |
| LEAQ (16*8)(aut), aut | |
| PSHUFB BSWAP, X0 | |
| PSHUFB BSWAP, X1 | |
| PSHUFB BSWAP, X2 | |
| PSHUFB BSWAP, X3 | |
| PSHUFB BSWAP, X4 | |
| PSHUFB BSWAP, X5 | |
| PSHUFB BSWAP, X6 | |
| PSHUFB BSWAP, X7 | |
| PXOR ACC0, X0 | |
| MOVOU (16*0)(pTbl), ACC0 | |
| MOVOU (16*1)(pTbl), ACCM | |
| MOVOU ACC0, ACC1 | |
| PSHUFD $78, X0, T1 | |
| PXOR X0, T1 | |
| PCLMULQDQ $0x00, X0, ACC0 | |
| PCLMULQDQ $0x11, X0, ACC1 | |
| PCLMULQDQ $0x00, T1, ACCM | |
| mulRoundAAD(X1, 1) | |
| mulRoundAAD(X2, 2) | |
| mulRoundAAD(X3, 3) | |
| mulRoundAAD(X4, 4) | |
| mulRoundAAD(X5, 5) | |
| mulRoundAAD(X6, 6) | |
| mulRoundAAD(X7, 7) | |
| PXOR ACC0, ACCM | |
| PXOR ACC1, ACCM | |
| MOVOU ACCM, T0 | |
| PSRLDQ $8, ACCM | |
| PSLLDQ $8, T0 | |
| PXOR ACCM, ACC1 | |
| PXOR T0, ACC0 | |
| reduceRound(ACC0) | |
| reduceRound(ACC0) | |
| PXOR ACC1, ACC0 | |
| JMP dataOctaLoop | |
| startSinglesLoop: | |
| MOVOU (16*14)(pTbl), T1 | |
| MOVOU (16*15)(pTbl), T2 | |
| dataSinglesLoop: | |
| CMPQ autLen, $16 | |
| JB dataEnd | |
| SUBQ $16, autLen | |
| MOVOU (aut), B0 | |
| dataMul: | |
| PSHUFB BSWAP, B0 | |
| PXOR ACC0, B0 | |
| MOVOU T1, ACC0 | |
| MOVOU T2, ACCM | |
| MOVOU T1, ACC1 | |
| PSHUFD $78, B0, T0 | |
| PXOR B0, T0 | |
| PCLMULQDQ $0x00, B0, ACC0 | |
| PCLMULQDQ $0x11, B0, ACC1 | |
| PCLMULQDQ $0x00, T0, ACCM | |
| PXOR ACC0, ACCM | |
| PXOR ACC1, ACCM | |
| MOVOU ACCM, T0 | |
| PSRLDQ $8, ACCM | |
| PSLLDQ $8, T0 | |
| PXOR ACCM, ACC1 | |
| PXOR T0, ACC0 | |
| MOVOU POLY, T0 | |
| PCLMULQDQ $0x01, ACC0, T0 | |
| PSHUFD $78, ACC0, ACC0 | |
| PXOR T0, ACC0 | |
| MOVOU POLY, T0 | |
| PCLMULQDQ $0x01, ACC0, T0 | |
| PSHUFD $78, ACC0, ACC0 | |
| PXOR T0, ACC0 | |
| PXOR ACC1, ACC0 | |
| LEAQ 16(aut), aut | |
| JMP dataSinglesLoop | |
| dataEnd: | |
| TESTQ autLen, autLen | |
| JEQ dataBail | |
| PXOR B0, B0 | |
| LEAQ -1(aut)(autLen*1), aut | |
| dataLoadLoop: | |
| PSLLDQ $1, B0 | |
| PINSRB $0, (aut), B0 | |
| LEAQ -1(aut), aut | |
| DECQ autLen | |
| JNE dataLoadLoop | |
| JMP dataMul | |
| dataBail: | |
| MOVOU ACC0, (tPtr) | |
| RET | |
| #undef pTbl | |
| #undef aut | |
| #undef tPtr | |
| #undef autLen | |
| // func gcmAesEnc(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32) | |
| TEXT ·gcmAesEnc(SB),0,$256-96 | |
| #define pTbl DI | |
| #define ctx DX | |
| #define ctrPtr CX | |
| #define ptx SI | |
| #define ks AX | |
| #define tPtr R8 | |
| #define ptxLen R9 | |
| #define aluCTR R10 | |
| #define aluTMP R11 | |
| #define aluK R12 | |
| #define NR R13 | |
| #define increment(i) ADDL $1, aluCTR; MOVL aluCTR, aluTMP; XORL aluK, aluTMP; BSWAPL aluTMP; MOVL aluTMP, (3*4 + 8*16 + i*16)(SP) | |
| #define aesRnd(k) AESENC k, B0; AESENC k, B1; AESENC k, B2; AESENC k, B3; AESENC k, B4; AESENC k, B5; AESENC k, B6; AESENC k, B7 | |
| #define aesRound(i) MOVOU (16*i)(ks), T0;AESENC T0, B0; AESENC T0, B1; AESENC T0, B2; AESENC T0, B3; AESENC T0, B4; AESENC T0, B5; AESENC T0, B6; AESENC T0, B7 | |
| #define aesRndLast(k) AESENCLAST k, B0; AESENCLAST k, B1; AESENCLAST k, B2; AESENCLAST k, B3; AESENCLAST k, B4; AESENCLAST k, B5; AESENCLAST k, B6; AESENCLAST k, B7 | |
| #define combinedRound(i) \ | |
| MOVOU (16*i)(ks), T0;\ | |
| AESENC T0, B0;\ | |
| AESENC T0, B1;\ | |
| AESENC T0, B2;\ | |
| AESENC T0, B3;\ | |
| MOVOU (16*(i*2))(pTbl), T1;\ | |
| MOVOU T1, T2;\ | |
| AESENC T0, B4;\ | |
| AESENC T0, B5;\ | |
| AESENC T0, B6;\ | |
| AESENC T0, B7;\ | |
| MOVOU (16*i)(SP), T0;\ | |
| PCLMULQDQ $0x00, T0, T1;\ | |
| PXOR T1, ACC0;\ | |
| PSHUFD $78, T0, T1;\ | |
| PCLMULQDQ $0x11, T0, T2;\ | |
| PXOR T1, T0;\ | |
| PXOR T2, ACC1;\ | |
| MOVOU (16*(i*2+1))(pTbl), T2;\ | |
| PCLMULQDQ $0x00, T2, T0;\ | |
| PXOR T0, ACCM | |
| #define mulRound(i) \ | |
| MOVOU (16*i)(SP), T0;\ | |
| MOVOU (16*(i*2))(pTbl), T1;\ | |
| MOVOU T1, T2;\ | |
| PCLMULQDQ $0x00, T0, T1;\ | |
| PXOR T1, ACC0;\ | |
| PCLMULQDQ $0x11, T0, T2;\ | |
| PXOR T2, ACC1;\ | |
| PSHUFD $78, T0, T1;\ | |
| PXOR T1, T0;\ | |
| MOVOU (16*(i*2+1))(pTbl), T1;\ | |
| PCLMULQDQ $0x00, T0, T1;\ | |
| PXOR T1, ACCM | |
| MOVQ productTable+0(FP), pTbl | |
| MOVQ dst+8(FP), ctx | |
| MOVQ src_base+32(FP), ptx | |
| MOVQ src_len+40(FP), ptxLen | |
| MOVQ ctr+56(FP), ctrPtr | |
| MOVQ T+64(FP), tPtr | |
| MOVQ ks_base+72(FP), ks | |
| MOVQ ks_len+80(FP), NR | |
| SHRQ $2, NR | |
| DECQ NR | |
| MOVOU bswapMask<>(SB), BSWAP | |
| MOVOU gcmPoly<>(SB), POLY | |
| MOVOU (tPtr), ACC0 | |
| PXOR ACC1, ACC1 | |
| PXOR ACCM, ACCM | |
| MOVOU (ctrPtr), B0 | |
| MOVL (3*4)(ctrPtr), aluCTR | |
| MOVOU (ks), T0 | |
| MOVL (3*4)(ks), aluK | |
| BSWAPL aluCTR | |
| BSWAPL aluK | |
| PXOR B0, T0 | |
| MOVOU T0, (8*16 + 0*16)(SP) | |
| increment(0) | |
| CMPQ ptxLen, $128 | |
| JB gcmAesEncSingles | |
| SUBQ $128, ptxLen | |
| // We have at least 8 blocks to encrypt, prepare the rest of the counters | |
| MOVOU T0, (8*16 + 1*16)(SP) | |
| increment(1) | |
| MOVOU T0, (8*16 + 2*16)(SP) | |
| increment(2) | |
| MOVOU T0, (8*16 + 3*16)(SP) | |
| increment(3) | |
| MOVOU T0, (8*16 + 4*16)(SP) | |
| increment(4) | |
| MOVOU T0, (8*16 + 5*16)(SP) | |
| increment(5) | |
| MOVOU T0, (8*16 + 6*16)(SP) | |
| increment(6) | |
| MOVOU T0, (8*16 + 7*16)(SP) | |
| increment(7) | |
| MOVOU (8*16 + 0*16)(SP), B0 | |
| MOVOU (8*16 + 1*16)(SP), B1 | |
| MOVOU (8*16 + 2*16)(SP), B2 | |
| MOVOU (8*16 + 3*16)(SP), B3 | |
| MOVOU (8*16 + 4*16)(SP), B4 | |
| MOVOU (8*16 + 5*16)(SP), B5 | |
| MOVOU (8*16 + 6*16)(SP), B6 | |
| MOVOU (8*16 + 7*16)(SP), B7 | |
| aesRound(1) | |
| increment(0) | |
| aesRound(2) | |
| increment(1) | |
| aesRound(3) | |
| increment(2) | |
| aesRound(4) | |
| increment(3) | |
| aesRound(5) | |
| increment(4) | |
| aesRound(6) | |
| increment(5) | |
| aesRound(7) | |
| increment(6) | |
| aesRound(8) | |
| increment(7) | |
| aesRound(9) | |
| MOVOU (16*10)(ks), T0 | |
| CMPQ NR, $12 | |
| JB encLast1 | |
| aesRnd(T0) | |
| aesRound(11) | |
| MOVOU (16*12)(ks), T0 | |
| JE encLast1 | |
| aesRnd(T0) | |
| aesRound(13) | |
| MOVOU (16*14)(ks), T0 | |
| encLast1: | |
| aesRndLast(T0) | |
| MOVOU (16*0)(ptx), T0 | |
| PXOR T0, B0 | |
| MOVOU (16*1)(ptx), T0 | |
| PXOR T0, B1 | |
| MOVOU (16*2)(ptx), T0 | |
| PXOR T0, B2 | |
| MOVOU (16*3)(ptx), T0 | |
| PXOR T0, B3 | |
| MOVOU (16*4)(ptx), T0 | |
| PXOR T0, B4 | |
| MOVOU (16*5)(ptx), T0 | |
| PXOR T0, B5 | |
| MOVOU (16*6)(ptx), T0 | |
| PXOR T0, B6 | |
| MOVOU (16*7)(ptx), T0 | |
| PXOR T0, B7 | |
| MOVOU B0, (16*0)(ctx) | |
| PSHUFB BSWAP, B0 | |
| PXOR ACC0, B0 | |
| MOVOU B1, (16*1)(ctx) | |
| PSHUFB BSWAP, B1 | |
| MOVOU B2, (16*2)(ctx) | |
| PSHUFB BSWAP, B2 | |
| MOVOU B3, (16*3)(ctx) | |
| PSHUFB BSWAP, B3 | |
| MOVOU B4, (16*4)(ctx) | |
| PSHUFB BSWAP, B4 | |
| MOVOU B5, (16*5)(ctx) | |
| PSHUFB BSWAP, B5 | |
| MOVOU B6, (16*6)(ctx) | |
| PSHUFB BSWAP, B6 | |
| MOVOU B7, (16*7)(ctx) | |
| PSHUFB BSWAP, B7 | |
| MOVOU B0, (16*0)(SP) | |
| MOVOU B1, (16*1)(SP) | |
| MOVOU B2, (16*2)(SP) | |
| MOVOU B3, (16*3)(SP) | |
| MOVOU B4, (16*4)(SP) | |
| MOVOU B5, (16*5)(SP) | |
| MOVOU B6, (16*6)(SP) | |
| MOVOU B7, (16*7)(SP) | |
| LEAQ 128(ptx), ptx | |
| LEAQ 128(ctx), ctx | |
| gcmAesEncOctetsLoop: | |
| CMPQ ptxLen, $128 | |
| JB gcmAesEncOctetsEnd | |
| SUBQ $128, ptxLen | |
| MOVOU (8*16 + 0*16)(SP), B0 | |
| MOVOU (8*16 + 1*16)(SP), B1 | |
| MOVOU (8*16 + 2*16)(SP), B2 | |
| MOVOU (8*16 + 3*16)(SP), B3 | |
| MOVOU (8*16 + 4*16)(SP), B4 | |
| MOVOU (8*16 + 5*16)(SP), B5 | |
| MOVOU (8*16 + 6*16)(SP), B6 | |
| MOVOU (8*16 + 7*16)(SP), B7 | |
| MOVOU (16*0)(SP), T0 | |
| PSHUFD $78, T0, T1 | |
| PXOR T0, T1 | |
| MOVOU (16*0)(pTbl), ACC0 | |
| MOVOU (16*1)(pTbl), ACCM | |
| MOVOU ACC0, ACC1 | |
| PCLMULQDQ $0x00, T1, ACCM | |
| PCLMULQDQ $0x00, T0, ACC0 | |
| PCLMULQDQ $0x11, T0, ACC1 | |
| combinedRound(1) | |
| increment(0) | |
| combinedRound(2) | |
| increment(1) | |
| combinedRound(3) | |
| increment(2) | |
| combinedRound(4) | |
| increment(3) | |
| combinedRound(5) | |
| increment(4) | |
| combinedRound(6) | |
| increment(5) | |
| combinedRound(7) | |
| increment(6) | |
| aesRound(8) | |
| increment(7) | |
| PXOR ACC0, ACCM | |
| PXOR ACC1, ACCM | |
| MOVOU ACCM, T0 | |
| PSRLDQ $8, ACCM | |
| PSLLDQ $8, T0 | |
| PXOR ACCM, ACC1 | |
| PXOR T0, ACC0 | |
| reduceRound(ACC0) | |
| aesRound(9) | |
| reduceRound(ACC0) | |
| PXOR ACC1, ACC0 | |
| MOVOU (16*10)(ks), T0 | |
| CMPQ NR, $12 | |
| JB encLast2 | |
| aesRnd(T0) | |
| aesRound(11) | |
| MOVOU (16*12)(ks), T0 | |
| JE encLast2 | |
| aesRnd(T0) | |
| aesRound(13) | |
| MOVOU (16*14)(ks), T0 | |
| encLast2: | |
| aesRndLast(T0) | |
| MOVOU (16*0)(ptx), T0 | |
| PXOR T0, B0 | |
| MOVOU (16*1)(ptx), T0 | |
| PXOR T0, B1 | |
| MOVOU (16*2)(ptx), T0 | |
| PXOR T0, B2 | |
| MOVOU (16*3)(ptx), T0 | |
| PXOR T0, B3 | |
| MOVOU (16*4)(ptx), T0 | |
| PXOR T0, B4 | |
| MOVOU (16*5)(ptx), T0 | |
| PXOR T0, B5 | |
| MOVOU (16*6)(ptx), T0 | |
| PXOR T0, B6 | |
| MOVOU (16*7)(ptx), T0 | |
| PXOR T0, B7 | |
| MOVOU B0, (16*0)(ctx) | |
| PSHUFB BSWAP, B0 | |
| PXOR ACC0, B0 | |
| MOVOU B1, (16*1)(ctx) | |
| PSHUFB BSWAP, B1 | |
| MOVOU B2, (16*2)(ctx) | |
| PSHUFB BSWAP, B2 | |
| MOVOU B3, (16*3)(ctx) | |
| PSHUFB BSWAP, B3 | |
| MOVOU B4, (16*4)(ctx) | |
| PSHUFB BSWAP, B4 | |
| MOVOU B5, (16*5)(ctx) | |
| PSHUFB BSWAP, B5 | |
| MOVOU B6, (16*6)(ctx) | |
| PSHUFB BSWAP, B6 | |
| MOVOU B7, (16*7)(ctx) | |
| PSHUFB BSWAP, B7 | |
| MOVOU B0, (16*0)(SP) | |
| MOVOU B1, (16*1)(SP) | |
| MOVOU B2, (16*2)(SP) | |
| MOVOU B3, (16*3)(SP) | |
| MOVOU B4, (16*4)(SP) | |
| MOVOU B5, (16*5)(SP) | |
| MOVOU B6, (16*6)(SP) | |
| MOVOU B7, (16*7)(SP) | |
| LEAQ 128(ptx), ptx | |
| LEAQ 128(ctx), ctx | |
| JMP gcmAesEncOctetsLoop | |
| gcmAesEncOctetsEnd: | |
| MOVOU (16*0)(SP), T0 | |
| MOVOU (16*0)(pTbl), ACC0 | |
| MOVOU (16*1)(pTbl), ACCM | |
| MOVOU ACC0, ACC1 | |
| PSHUFD $78, T0, T1 | |
| PXOR T0, T1 | |
| PCLMULQDQ $0x00, T0, ACC0 | |
| PCLMULQDQ $0x11, T0, ACC1 | |
| PCLMULQDQ $0x00, T1, ACCM | |
| mulRound(1) | |
| mulRound(2) | |
| mulRound(3) | |
| mulRound(4) | |
| mulRound(5) | |
| mulRound(6) | |
| mulRound(7) | |
| PXOR ACC0, ACCM | |
| PXOR ACC1, ACCM | |
| MOVOU ACCM, T0 | |
| PSRLDQ $8, ACCM | |
| PSLLDQ $8, T0 | |
| PXOR ACCM, ACC1 | |
| PXOR T0, ACC0 | |
| reduceRound(ACC0) | |
| reduceRound(ACC0) | |
| PXOR ACC1, ACC0 | |
| TESTQ ptxLen, ptxLen | |
| JE gcmAesEncDone | |
| SUBQ $7, aluCTR | |
| gcmAesEncSingles: | |
| MOVOU (16*1)(ks), B1 | |
| MOVOU (16*2)(ks), B2 | |
| MOVOU (16*3)(ks), B3 | |
| MOVOU (16*4)(ks), B4 | |
| MOVOU (16*5)(ks), B5 | |
| MOVOU (16*6)(ks), B6 | |
| MOVOU (16*7)(ks), B7 | |
| MOVOU (16*14)(pTbl), T2 | |
| gcmAesEncSinglesLoop: | |
| CMPQ ptxLen, $16 | |
| JB gcmAesEncTail | |
| SUBQ $16, ptxLen | |
| MOVOU (8*16 + 0*16)(SP), B0 | |
| increment(0) | |
| AESENC B1, B0 | |
| AESENC B2, B0 | |
| AESENC B3, B0 | |
| AESENC B4, B0 | |
| AESENC B5, B0 | |
| AESENC B6, B0 | |
| AESENC B7, B0 | |
| MOVOU (16*8)(ks), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*9)(ks), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*10)(ks), T0 | |
| CMPQ NR, $12 | |
| JB encLast3 | |
| AESENC T0, B0 | |
| MOVOU (16*11)(ks), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*12)(ks), T0 | |
| JE encLast3 | |
| AESENC T0, B0 | |
| MOVOU (16*13)(ks), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*14)(ks), T0 | |
| encLast3: | |
| AESENCLAST T0, B0 | |
| MOVOU (ptx), T0 | |
| PXOR T0, B0 | |
| MOVOU B0, (ctx) | |
| PSHUFB BSWAP, B0 | |
| PXOR ACC0, B0 | |
| MOVOU T2, ACC0 | |
| MOVOU T2, ACC1 | |
| MOVOU (16*15)(pTbl), ACCM | |
| PSHUFD $78, B0, T0 | |
| PXOR B0, T0 | |
| PCLMULQDQ $0x00, B0, ACC0 | |
| PCLMULQDQ $0x11, B0, ACC1 | |
| PCLMULQDQ $0x00, T0, ACCM | |
| PXOR ACC0, ACCM | |
| PXOR ACC1, ACCM | |
| MOVOU ACCM, T0 | |
| PSRLDQ $8, ACCM | |
| PSLLDQ $8, T0 | |
| PXOR ACCM, ACC1 | |
| PXOR T0, ACC0 | |
| reduceRound(ACC0) | |
| reduceRound(ACC0) | |
| PXOR ACC1, ACC0 | |
| LEAQ (16*1)(ptx), ptx | |
| LEAQ (16*1)(ctx), ctx | |
| JMP gcmAesEncSinglesLoop | |
| gcmAesEncTail: | |
| TESTQ ptxLen, ptxLen | |
| JE gcmAesEncDone | |
| MOVOU (8*16 + 0*16)(SP), B0 | |
| AESENC B1, B0 | |
| AESENC B2, B0 | |
| AESENC B3, B0 | |
| AESENC B4, B0 | |
| AESENC B5, B0 | |
| AESENC B6, B0 | |
| AESENC B7, B0 | |
| MOVOU (16*8)(ks), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*9)(ks), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*10)(ks), T0 | |
| CMPQ NR, $12 | |
| JB encLast4 | |
| AESENC T0, B0 | |
| MOVOU (16*11)(ks), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*12)(ks), T0 | |
| JE encLast4 | |
| AESENC T0, B0 | |
| MOVOU (16*13)(ks), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*14)(ks), T0 | |
| encLast4: | |
| AESENCLAST T0, B0 | |
| MOVOU B0, T0 | |
| LEAQ -1(ptx)(ptxLen*1), ptx | |
| MOVQ ptxLen, aluTMP | |
| SHLQ $4, aluTMP | |
| LEAQ andMask<>(SB), aluCTR | |
| MOVOU -16(aluCTR)(aluTMP*1), T1 | |
| PXOR B0, B0 | |
| ptxLoadLoop: | |
| PSLLDQ $1, B0 | |
| PINSRB $0, (ptx), B0 | |
| LEAQ -1(ptx), ptx | |
| DECQ ptxLen | |
| JNE ptxLoadLoop | |
| PXOR T0, B0 | |
| PAND T1, B0 | |
| MOVOU B0, (ctx) // I assume there is always space, due to TAG in the end of the CT | |
| PSHUFB BSWAP, B0 | |
| PXOR ACC0, B0 | |
| MOVOU T2, ACC0 | |
| MOVOU T2, ACC1 | |
| MOVOU (16*15)(pTbl), ACCM | |
| PSHUFD $78, B0, T0 | |
| PXOR B0, T0 | |
| PCLMULQDQ $0x00, B0, ACC0 | |
| PCLMULQDQ $0x11, B0, ACC1 | |
| PCLMULQDQ $0x00, T0, ACCM | |
| PXOR ACC0, ACCM | |
| PXOR ACC1, ACCM | |
| MOVOU ACCM, T0 | |
| PSRLDQ $8, ACCM | |
| PSLLDQ $8, T0 | |
| PXOR ACCM, ACC1 | |
| PXOR T0, ACC0 | |
| reduceRound(ACC0) | |
| reduceRound(ACC0) | |
| PXOR ACC1, ACC0 | |
| gcmAesEncDone: | |
| MOVOU ACC0, (tPtr) | |
| RET | |
| #undef increment | |
| // func gcmAesDec(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32) | |
| TEXT ·gcmAesDec(SB),0,$128-96 | |
| #define increment(i) ADDL $1, aluCTR; MOVL aluCTR, aluTMP; XORL aluK, aluTMP; BSWAPL aluTMP; MOVL aluTMP, (3*4 + i*16)(SP) | |
| #define combinedDecRound(i) \ | |
| MOVOU (16*i)(ks), T0;\ | |
| AESENC T0, B0;\ | |
| AESENC T0, B1;\ | |
| AESENC T0, B2;\ | |
| AESENC T0, B3;\ | |
| MOVOU (16*(i*2))(pTbl), T1;\ | |
| MOVOU T1, T2;\ | |
| AESENC T0, B4;\ | |
| AESENC T0, B5;\ | |
| AESENC T0, B6;\ | |
| AESENC T0, B7;\ | |
| MOVOU (16*i)(ctx), T0;\ | |
| PSHUFB BSWAP, T0;\ | |
| PCLMULQDQ $0x00, T0, T1;\ | |
| PXOR T1, ACC0;\ | |
| PSHUFD $78, T0, T1;\ | |
| PCLMULQDQ $0x11, T0, T2;\ | |
| PXOR T1, T0;\ | |
| PXOR T2, ACC1;\ | |
| MOVOU (16*(i*2+1))(pTbl), T2;\ | |
| PCLMULQDQ $0x00, T2, T0;\ | |
| PXOR T0, ACCM | |
| MOVQ productTable+0(FP), pTbl | |
| MOVQ dst+8(FP), ptx | |
| MOVQ src_base+32(FP), ctx | |
| MOVQ src_len+40(FP), ptxLen | |
| MOVQ ctr+56(FP), ctrPtr | |
| MOVQ T+64(FP), tPtr | |
| MOVQ ks_base+72(FP), ks | |
| MOVQ ks_len+80(FP), NR | |
| SHRQ $2, NR | |
| DECQ NR | |
| MOVOU bswapMask<>(SB), BSWAP | |
| MOVOU gcmPoly<>(SB), POLY | |
| MOVOU (tPtr), ACC0 | |
| PXOR ACC1, ACC1 | |
| PXOR ACCM, ACCM | |
| MOVOU (ctrPtr), B0 | |
| MOVL (3*4)(ctrPtr), aluCTR | |
| MOVOU (ks), T0 | |
| MOVL (3*4)(ks), aluK | |
| BSWAPL aluCTR | |
| BSWAPL aluK | |
| PXOR B0, T0 | |
| MOVOU T0, (0*16)(SP) | |
| increment(0) | |
| CMPQ ptxLen, $128 | |
| JB gcmAesDecSingles | |
| MOVOU T0, (1*16)(SP) | |
| increment(1) | |
| MOVOU T0, (2*16)(SP) | |
| increment(2) | |
| MOVOU T0, (3*16)(SP) | |
| increment(3) | |
| MOVOU T0, (4*16)(SP) | |
| increment(4) | |
| MOVOU T0, (5*16)(SP) | |
| increment(5) | |
| MOVOU T0, (6*16)(SP) | |
| increment(6) | |
| MOVOU T0, (7*16)(SP) | |
| increment(7) | |
| gcmAesDecOctetsLoop: | |
| CMPQ ptxLen, $128 | |
| JB gcmAesDecEndOctets | |
| SUBQ $128, ptxLen | |
| MOVOU (0*16)(SP), B0 | |
| MOVOU (1*16)(SP), B1 | |
| MOVOU (2*16)(SP), B2 | |
| MOVOU (3*16)(SP), B3 | |
| MOVOU (4*16)(SP), B4 | |
| MOVOU (5*16)(SP), B5 | |
| MOVOU (6*16)(SP), B6 | |
| MOVOU (7*16)(SP), B7 | |
| MOVOU (16*0)(ctx), T0 | |
| PSHUFB BSWAP, T0 | |
| PXOR ACC0, T0 | |
| PSHUFD $78, T0, T1 | |
| PXOR T0, T1 | |
| MOVOU (16*0)(pTbl), ACC0 | |
| MOVOU (16*1)(pTbl), ACCM | |
| MOVOU ACC0, ACC1 | |
| PCLMULQDQ $0x00, T1, ACCM | |
| PCLMULQDQ $0x00, T0, ACC0 | |
| PCLMULQDQ $0x11, T0, ACC1 | |
| combinedDecRound(1) | |
| increment(0) | |
| combinedDecRound(2) | |
| increment(1) | |
| combinedDecRound(3) | |
| increment(2) | |
| combinedDecRound(4) | |
| increment(3) | |
| combinedDecRound(5) | |
| increment(4) | |
| combinedDecRound(6) | |
| increment(5) | |
| combinedDecRound(7) | |
| increment(6) | |
| aesRound(8) | |
| increment(7) | |
| PXOR ACC0, ACCM | |
| PXOR ACC1, ACCM | |
| MOVOU ACCM, T0 | |
| PSRLDQ $8, ACCM | |
| PSLLDQ $8, T0 | |
| PXOR ACCM, ACC1 | |
| PXOR T0, ACC0 | |
| reduceRound(ACC0) | |
| aesRound(9) | |
| reduceRound(ACC0) | |
| PXOR ACC1, ACC0 | |
| MOVOU (16*10)(ks), T0 | |
| CMPQ NR, $12 | |
| JB decLast1 | |
| aesRnd(T0) | |
| aesRound(11) | |
| MOVOU (16*12)(ks), T0 | |
| JE decLast1 | |
| aesRnd(T0) | |
| aesRound(13) | |
| MOVOU (16*14)(ks), T0 | |
| decLast1: | |
| aesRndLast(T0) | |
| MOVOU (16*0)(ctx), T0 | |
| PXOR T0, B0 | |
| MOVOU (16*1)(ctx), T0 | |
| PXOR T0, B1 | |
| MOVOU (16*2)(ctx), T0 | |
| PXOR T0, B2 | |
| MOVOU (16*3)(ctx), T0 | |
| PXOR T0, B3 | |
| MOVOU (16*4)(ctx), T0 | |
| PXOR T0, B4 | |
| MOVOU (16*5)(ctx), T0 | |
| PXOR T0, B5 | |
| MOVOU (16*6)(ctx), T0 | |
| PXOR T0, B6 | |
| MOVOU (16*7)(ctx), T0 | |
| PXOR T0, B7 | |
| MOVOU B0, (16*0)(ptx) | |
| MOVOU B1, (16*1)(ptx) | |
| MOVOU B2, (16*2)(ptx) | |
| MOVOU B3, (16*3)(ptx) | |
| MOVOU B4, (16*4)(ptx) | |
| MOVOU B5, (16*5)(ptx) | |
| MOVOU B6, (16*6)(ptx) | |
| MOVOU B7, (16*7)(ptx) | |
| LEAQ 128(ptx), ptx | |
| LEAQ 128(ctx), ctx | |
| JMP gcmAesDecOctetsLoop | |
| gcmAesDecEndOctets: | |
| SUBQ $7, aluCTR | |
| gcmAesDecSingles: | |
| MOVOU (16*1)(ks), B1 | |
| MOVOU (16*2)(ks), B2 | |
| MOVOU (16*3)(ks), B3 | |
| MOVOU (16*4)(ks), B4 | |
| MOVOU (16*5)(ks), B5 | |
| MOVOU (16*6)(ks), B6 | |
| MOVOU (16*7)(ks), B7 | |
| MOVOU (16*14)(pTbl), T2 | |
| gcmAesDecSinglesLoop: | |
| CMPQ ptxLen, $16 | |
| JB gcmAesDecTail | |
| SUBQ $16, ptxLen | |
| MOVOU (ctx), B0 | |
| MOVOU B0, T1 | |
| PSHUFB BSWAP, B0 | |
| PXOR ACC0, B0 | |
| MOVOU T2, ACC0 | |
| MOVOU T2, ACC1 | |
| MOVOU (16*15)(pTbl), ACCM | |
| PCLMULQDQ $0x00, B0, ACC0 | |
| PCLMULQDQ $0x11, B0, ACC1 | |
| PSHUFD $78, B0, T0 | |
| PXOR B0, T0 | |
| PCLMULQDQ $0x00, T0, ACCM | |
| PXOR ACC0, ACCM | |
| PXOR ACC1, ACCM | |
| MOVOU ACCM, T0 | |
| PSRLDQ $8, ACCM | |
| PSLLDQ $8, T0 | |
| PXOR ACCM, ACC1 | |
| PXOR T0, ACC0 | |
| reduceRound(ACC0) | |
| reduceRound(ACC0) | |
| PXOR ACC1, ACC0 | |
| MOVOU (0*16)(SP), B0 | |
| increment(0) | |
| AESENC B1, B0 | |
| AESENC B2, B0 | |
| AESENC B3, B0 | |
| AESENC B4, B0 | |
| AESENC B5, B0 | |
| AESENC B6, B0 | |
| AESENC B7, B0 | |
| MOVOU (16*8)(ks), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*9)(ks), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*10)(ks), T0 | |
| CMPQ NR, $12 | |
| JB decLast2 | |
| AESENC T0, B0 | |
| MOVOU (16*11)(ks), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*12)(ks), T0 | |
| JE decLast2 | |
| AESENC T0, B0 | |
| MOVOU (16*13)(ks), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*14)(ks), T0 | |
| decLast2: | |
| AESENCLAST T0, B0 | |
| PXOR T1, B0 | |
| MOVOU B0, (ptx) | |
| LEAQ (16*1)(ptx), ptx | |
| LEAQ (16*1)(ctx), ctx | |
| JMP gcmAesDecSinglesLoop | |
| gcmAesDecTail: | |
| TESTQ ptxLen, ptxLen | |
| JE gcmAesDecDone | |
| MOVQ ptxLen, aluTMP | |
| SHLQ $4, aluTMP | |
| LEAQ andMask<>(SB), aluCTR | |
| MOVOU -16(aluCTR)(aluTMP*1), T1 | |
| MOVOU (ctx), B0 // I assume there is TAG attached to the ctx, and there is no read overflow | |
| PAND T1, B0 | |
| MOVOU B0, T1 | |
| PSHUFB BSWAP, B0 | |
| PXOR ACC0, B0 | |
| MOVOU (16*14)(pTbl), ACC0 | |
| MOVOU (16*15)(pTbl), ACCM | |
| MOVOU ACC0, ACC1 | |
| PCLMULQDQ $0x00, B0, ACC0 | |
| PCLMULQDQ $0x11, B0, ACC1 | |
| PSHUFD $78, B0, T0 | |
| PXOR B0, T0 | |
| PCLMULQDQ $0x00, T0, ACCM | |
| PXOR ACC0, ACCM | |
| PXOR ACC1, ACCM | |
| MOVOU ACCM, T0 | |
| PSRLDQ $8, ACCM | |
| PSLLDQ $8, T0 | |
| PXOR ACCM, ACC1 | |
| PXOR T0, ACC0 | |
| reduceRound(ACC0) | |
| reduceRound(ACC0) | |
| PXOR ACC1, ACC0 | |
| MOVOU (0*16)(SP), B0 | |
| increment(0) | |
| AESENC B1, B0 | |
| AESENC B2, B0 | |
| AESENC B3, B0 | |
| AESENC B4, B0 | |
| AESENC B5, B0 | |
| AESENC B6, B0 | |
| AESENC B7, B0 | |
| MOVOU (16*8)(ks), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*9)(ks), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*10)(ks), T0 | |
| CMPQ NR, $12 | |
| JB decLast3 | |
| AESENC T0, B0 | |
| MOVOU (16*11)(ks), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*12)(ks), T0 | |
| JE decLast3 | |
| AESENC T0, B0 | |
| MOVOU (16*13)(ks), T0 | |
| AESENC T0, B0 | |
| MOVOU (16*14)(ks), T0 | |
| decLast3: | |
| AESENCLAST T0, B0 | |
| PXOR T1, B0 | |
| ptxStoreLoop: | |
| PEXTRB $0, B0, (ptx) | |
| PSRLDQ $1, B0 | |
| LEAQ 1(ptx), ptx | |
| DECQ ptxLen | |
| JNE ptxStoreLoop | |
| gcmAesDecDone: | |
| MOVOU ACC0, (tPtr) | |
| RET |