| // Copyright 2018 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| #include "textflag.h" |
| |
| #define B0 V0 |
| #define B1 V1 |
| #define B2 V2 |
| #define B3 V3 |
| #define B4 V4 |
| #define B5 V5 |
| #define B6 V6 |
| #define B7 V7 |
| |
| #define ACC0 V8 |
| #define ACC1 V9 |
| #define ACCM V10 |
| |
| #define T0 V11 |
| #define T1 V12 |
| #define T2 V13 |
| #define T3 V14 |
| |
| #define POLY V15 |
| #define ZERO V16 |
| #define INC V17 |
| #define CTR V18 |
| |
| #define K0 V19 |
| #define K1 V20 |
| #define K2 V21 |
| #define K3 V22 |
| #define K4 V23 |
| #define K5 V24 |
| #define K6 V25 |
| #define K7 V26 |
| #define K8 V27 |
| #define K9 V28 |
| #define K10 V29 |
| #define K11 V30 |
| #define KLAST V31 |
| |
| #define reduce() \ |
| VEOR ACC0.B16, ACCM.B16, ACCM.B16 \ |
| VEOR ACC1.B16, ACCM.B16, ACCM.B16 \ |
| VEXT $8, ZERO.B16, ACCM.B16, T0.B16 \ |
| VEXT $8, ACCM.B16, ZERO.B16, ACCM.B16 \ |
| VEOR ACCM.B16, ACC0.B16, ACC0.B16 \ |
| VEOR T0.B16, ACC1.B16, ACC1.B16 \ |
| VPMULL POLY.D1, ACC0.D1, T0.Q1 \ |
| VEXT $8, ACC0.B16, ACC0.B16, ACC0.B16 \ |
| VEOR T0.B16, ACC0.B16, ACC0.B16 \ |
| VPMULL POLY.D1, ACC0.D1, T0.Q1 \ |
| VEOR T0.B16, ACC1.B16, ACC1.B16 \ |
| VEXT $8, ACC1.B16, ACC1.B16, ACC1.B16 \ |
| VEOR ACC1.B16, ACC0.B16, ACC0.B16 \ |
| |
| // func gcmAesFinish(productTable *[256]byte, tagMask, T *[16]byte, pLen, dLen uint64) |
| TEXT ·gcmAesFinish(SB),NOSPLIT,$0 |
| #define pTbl R0 |
| #define tMsk R1 |
| #define tPtr R2 |
| #define plen R3 |
| #define dlen R4 |
| |
| MOVD $0xC2, R1 |
| LSL $56, R1 |
| MOVD $1, R0 |
| VMOV R1, POLY.D[0] |
| VMOV R0, POLY.D[1] |
| VEOR ZERO.B16, ZERO.B16, ZERO.B16 |
| |
| MOVD productTable+0(FP), pTbl |
| MOVD tagMask+8(FP), tMsk |
| MOVD T+16(FP), tPtr |
| MOVD pLen+24(FP), plen |
| MOVD dLen+32(FP), dlen |
| |
| VLD1 (tPtr), [ACC0.B16] |
| VLD1 (tMsk), [B1.B16] |
| |
| LSL $3, plen |
| LSL $3, dlen |
| |
| VMOV dlen, B0.D[0] |
| VMOV plen, B0.D[1] |
| |
| ADD $14*16, pTbl |
| VLD1.P (pTbl), [T1.B16, T2.B16] |
| |
| VEOR ACC0.B16, B0.B16, B0.B16 |
| |
| VEXT $8, B0.B16, B0.B16, T0.B16 |
| VEOR B0.B16, T0.B16, T0.B16 |
| VPMULL B0.D1, T1.D1, ACC1.Q1 |
| VPMULL2 B0.D2, T1.D2, ACC0.Q1 |
| VPMULL T0.D1, T2.D1, ACCM.Q1 |
| |
| reduce() |
| |
| VREV64 ACC0.B16, ACC0.B16 |
| VEOR B1.B16, ACC0.B16, ACC0.B16 |
| |
| VST1 [ACC0.B16], (tPtr) |
| RET |
| #undef pTbl |
| #undef tMsk |
| #undef tPtr |
| #undef plen |
| #undef dlen |
| |
| // func gcmAesInit(productTable *[256]byte, ks []uint32) |
| TEXT ·gcmAesInit(SB),NOSPLIT,$0 |
| #define pTbl R0 |
| #define KS R1 |
| #define NR R2 |
| #define I R3 |
| MOVD productTable+0(FP), pTbl |
| MOVD ks_base+8(FP), KS |
| MOVD ks_len+16(FP), NR |
| |
| MOVD $0xC2, I |
| LSL $56, I |
| VMOV I, POLY.D[0] |
| MOVD $1, I |
| VMOV I, POLY.D[1] |
| VEOR ZERO.B16, ZERO.B16, ZERO.B16 |
| |
| // Encrypt block 0 with the AES key to generate the hash key H |
| VLD1.P 64(KS), [T0.B16, T1.B16, T2.B16, T3.B16] |
| VEOR B0.B16, B0.B16, B0.B16 |
| AESE T0.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE T1.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE T2.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE T3.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| VLD1.P 64(KS), [T0.B16, T1.B16, T2.B16, T3.B16] |
| AESE T0.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE T1.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE T2.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE T3.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| TBZ $4, NR, initEncFinish |
| VLD1.P 32(KS), [T0.B16, T1.B16] |
| AESE T0.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE T1.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| TBZ $3, NR, initEncFinish |
| VLD1.P 32(KS), [T0.B16, T1.B16] |
| AESE T0.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE T1.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| initEncFinish: |
| VLD1 (KS), [T0.B16, T1.B16, T2.B16] |
| AESE T0.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE T1.B16, B0.B16 |
| VEOR T2.B16, B0.B16, B0.B16 |
| |
| VREV64 B0.B16, B0.B16 |
| |
| // Multiply by 2 modulo P |
| VMOV B0.D[0], I |
| ASR $63, I |
| VMOV I, T1.D[0] |
| VMOV I, T1.D[1] |
| VAND POLY.B16, T1.B16, T1.B16 |
| VUSHR $63, B0.D2, T2.D2 |
| VEXT $8, ZERO.B16, T2.B16, T2.B16 |
| VSHL $1, B0.D2, B0.D2 |
| VEOR T1.B16, B0.B16, B0.B16 |
| VEOR T2.B16, B0.B16, B0.B16 // Can avoid this when VSLI is available |
| |
| // Karatsuba pre-computation |
| VEXT $8, B0.B16, B0.B16, B1.B16 |
| VEOR B0.B16, B1.B16, B1.B16 |
| |
| ADD $14*16, pTbl |
| VST1 [B0.B16, B1.B16], (pTbl) |
| SUB $2*16, pTbl |
| |
| VMOV B0.B16, B2.B16 |
| VMOV B1.B16, B3.B16 |
| |
| MOVD $7, I |
| |
| initLoop: |
| // Compute powers of H |
| SUBS $1, I |
| |
| VPMULL B0.D1, B2.D1, T1.Q1 |
| VPMULL2 B0.D2, B2.D2, T0.Q1 |
| VPMULL B1.D1, B3.D1, T2.Q1 |
| VEOR T0.B16, T2.B16, T2.B16 |
| VEOR T1.B16, T2.B16, T2.B16 |
| VEXT $8, ZERO.B16, T2.B16, T3.B16 |
| VEXT $8, T2.B16, ZERO.B16, T2.B16 |
| VEOR T2.B16, T0.B16, T0.B16 |
| VEOR T3.B16, T1.B16, T1.B16 |
| VPMULL POLY.D1, T0.D1, T2.Q1 |
| VEXT $8, T0.B16, T0.B16, T0.B16 |
| VEOR T2.B16, T0.B16, T0.B16 |
| VPMULL POLY.D1, T0.D1, T2.Q1 |
| VEXT $8, T0.B16, T0.B16, T0.B16 |
| VEOR T2.B16, T0.B16, T0.B16 |
| VEOR T1.B16, T0.B16, B2.B16 |
| VMOV B2.B16, B3.B16 |
| VEXT $8, B2.B16, B2.B16, B2.B16 |
| VEOR B2.B16, B3.B16, B3.B16 |
| |
| VST1 [B2.B16, B3.B16], (pTbl) |
| SUB $2*16, pTbl |
| |
| BNE initLoop |
| RET |
| #undef I |
| #undef NR |
| #undef KS |
| #undef pTbl |
| |
| // func gcmAesData(productTable *[256]byte, data []byte, T *[16]byte) |
| TEXT ·gcmAesData(SB),NOSPLIT,$0 |
| #define pTbl R0 |
| #define aut R1 |
| #define tPtr R2 |
| #define autLen R3 |
| #define H0 R4 |
| #define pTblSave R5 |
| |
| #define mulRound(X) \ |
| VLD1.P 32(pTbl), [T1.B16, T2.B16] \ |
| VREV64 X.B16, X.B16 \ |
| VEXT $8, X.B16, X.B16, T0.B16 \ |
| VEOR X.B16, T0.B16, T0.B16 \ |
| VPMULL X.D1, T1.D1, T3.Q1 \ |
| VEOR T3.B16, ACC1.B16, ACC1.B16 \ |
| VPMULL2 X.D2, T1.D2, T3.Q1 \ |
| VEOR T3.B16, ACC0.B16, ACC0.B16 \ |
| VPMULL T0.D1, T2.D1, T3.Q1 \ |
| VEOR T3.B16, ACCM.B16, ACCM.B16 |
| |
| MOVD productTable+0(FP), pTbl |
| MOVD data_base+8(FP), aut |
| MOVD data_len+16(FP), autLen |
| MOVD T+32(FP), tPtr |
| |
| VEOR ACC0.B16, ACC0.B16, ACC0.B16 |
| CBZ autLen, dataBail |
| |
| MOVD $0xC2, H0 |
| LSL $56, H0 |
| VMOV H0, POLY.D[0] |
| MOVD $1, H0 |
| VMOV H0, POLY.D[1] |
| VEOR ZERO.B16, ZERO.B16, ZERO.B16 |
| MOVD pTbl, pTblSave |
| |
| CMP $13, autLen |
| BEQ dataTLS |
| CMP $128, autLen |
| BLT startSinglesLoop |
| B octetsLoop |
| |
| dataTLS: |
| ADD $14*16, pTbl |
| VLD1.P (pTbl), [T1.B16, T2.B16] |
| VEOR B0.B16, B0.B16, B0.B16 |
| |
| MOVD (aut), H0 |
| VMOV H0, B0.D[0] |
| MOVW 8(aut), H0 |
| VMOV H0, B0.S[2] |
| MOVB 12(aut), H0 |
| VMOV H0, B0.B[12] |
| |
| MOVD $0, autLen |
| B dataMul |
| |
| octetsLoop: |
| CMP $128, autLen |
| BLT startSinglesLoop |
| SUB $128, autLen |
| |
| VLD1.P 32(aut), [B0.B16, B1.B16] |
| |
| VLD1.P 32(pTbl), [T1.B16, T2.B16] |
| VREV64 B0.B16, B0.B16 |
| VEOR ACC0.B16, B0.B16, B0.B16 |
| VEXT $8, B0.B16, B0.B16, T0.B16 |
| VEOR B0.B16, T0.B16, T0.B16 |
| VPMULL B0.D1, T1.D1, ACC1.Q1 |
| VPMULL2 B0.D2, T1.D2, ACC0.Q1 |
| VPMULL T0.D1, T2.D1, ACCM.Q1 |
| |
| mulRound(B1) |
| VLD1.P 32(aut), [B2.B16, B3.B16] |
| mulRound(B2) |
| mulRound(B3) |
| VLD1.P 32(aut), [B4.B16, B5.B16] |
| mulRound(B4) |
| mulRound(B5) |
| VLD1.P 32(aut), [B6.B16, B7.B16] |
| mulRound(B6) |
| mulRound(B7) |
| |
| MOVD pTblSave, pTbl |
| reduce() |
| B octetsLoop |
| |
| startSinglesLoop: |
| |
| ADD $14*16, pTbl |
| VLD1.P (pTbl), [T1.B16, T2.B16] |
| |
| singlesLoop: |
| |
| CMP $16, autLen |
| BLT dataEnd |
| SUB $16, autLen |
| |
| VLD1.P 16(aut), [B0.B16] |
| dataMul: |
| VREV64 B0.B16, B0.B16 |
| VEOR ACC0.B16, B0.B16, B0.B16 |
| |
| VEXT $8, B0.B16, B0.B16, T0.B16 |
| VEOR B0.B16, T0.B16, T0.B16 |
| VPMULL B0.D1, T1.D1, ACC1.Q1 |
| VPMULL2 B0.D2, T1.D2, ACC0.Q1 |
| VPMULL T0.D1, T2.D1, ACCM.Q1 |
| |
| reduce() |
| |
| B singlesLoop |
| |
| dataEnd: |
| |
| CBZ autLen, dataBail |
| VEOR B0.B16, B0.B16, B0.B16 |
| ADD autLen, aut |
| |
| dataLoadLoop: |
| MOVB.W -1(aut), H0 |
| VEXT $15, B0.B16, ZERO.B16, B0.B16 |
| VMOV H0, B0.B[0] |
| SUBS $1, autLen |
| BNE dataLoadLoop |
| B dataMul |
| |
| dataBail: |
| VST1 [ACC0.B16], (tPtr) |
| RET |
| |
| #undef pTbl |
| #undef aut |
| #undef tPtr |
| #undef autLen |
| #undef H0 |
| #undef pTblSave |
| |
| // func gcmAesEnc(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32) |
| TEXT ·gcmAesEnc(SB),NOSPLIT,$0 |
| #define pTbl R0 |
| #define dstPtr R1 |
| #define ctrPtr R2 |
| #define srcPtr R3 |
| #define ks R4 |
| #define tPtr R5 |
| #define srcPtrLen R6 |
| #define aluCTR R7 |
| #define aluTMP R8 |
| #define aluK R9 |
| #define NR R10 |
| #define H0 R11 |
| #define H1 R12 |
| #define curK R13 |
| #define pTblSave R14 |
| |
| #define aesrndx8(K) \ |
| AESE K.B16, B0.B16 \ |
| AESMC B0.B16, B0.B16 \ |
| AESE K.B16, B1.B16 \ |
| AESMC B1.B16, B1.B16 \ |
| AESE K.B16, B2.B16 \ |
| AESMC B2.B16, B2.B16 \ |
| AESE K.B16, B3.B16 \ |
| AESMC B3.B16, B3.B16 \ |
| AESE K.B16, B4.B16 \ |
| AESMC B4.B16, B4.B16 \ |
| AESE K.B16, B5.B16 \ |
| AESMC B5.B16, B5.B16 \ |
| AESE K.B16, B6.B16 \ |
| AESMC B6.B16, B6.B16 \ |
| AESE K.B16, B7.B16 \ |
| AESMC B7.B16, B7.B16 |
| |
| #define aesrndlastx8(K) \ |
| AESE K.B16, B0.B16 \ |
| AESE K.B16, B1.B16 \ |
| AESE K.B16, B2.B16 \ |
| AESE K.B16, B3.B16 \ |
| AESE K.B16, B4.B16 \ |
| AESE K.B16, B5.B16 \ |
| AESE K.B16, B6.B16 \ |
| AESE K.B16, B7.B16 |
| |
| MOVD productTable+0(FP), pTbl |
| MOVD dst+8(FP), dstPtr |
| MOVD src_base+32(FP), srcPtr |
| MOVD src_len+40(FP), srcPtrLen |
| MOVD ctr+56(FP), ctrPtr |
| MOVD T+64(FP), tPtr |
| MOVD ks_base+72(FP), ks |
| MOVD ks_len+80(FP), NR |
| |
| MOVD $0xC2, H1 |
| LSL $56, H1 |
| MOVD $1, H0 |
| VMOV H1, POLY.D[0] |
| VMOV H0, POLY.D[1] |
| VEOR ZERO.B16, ZERO.B16, ZERO.B16 |
| // Compute NR from len(ks) |
| MOVD pTbl, pTblSave |
| // Current tag, after AAD |
| VLD1 (tPtr), [ACC0.B16] |
| VEOR ACC1.B16, ACC1.B16, ACC1.B16 |
| VEOR ACCM.B16, ACCM.B16, ACCM.B16 |
| // Prepare initial counter, and the increment vector |
| VLD1 (ctrPtr), [CTR.B16] |
| VEOR INC.B16, INC.B16, INC.B16 |
| MOVD $1, H0 |
| VMOV H0, INC.S[3] |
| VREV32 CTR.B16, CTR.B16 |
| VADD CTR.S4, INC.S4, CTR.S4 |
| // Skip to <8 blocks loop |
| CMP $128, srcPtrLen |
| |
| MOVD ks, H0 |
| // For AES-128 round keys are stored in: K0 .. K10, KLAST |
| VLD1.P 64(H0), [K0.B16, K1.B16, K2.B16, K3.B16] |
| VLD1.P 64(H0), [K4.B16, K5.B16, K6.B16, K7.B16] |
| VLD1.P 48(H0), [K8.B16, K9.B16, K10.B16] |
| VMOV K10.B16, KLAST.B16 |
| |
| BLT startSingles |
| // There are at least 8 blocks to encrypt |
| TBZ $4, NR, octetsLoop |
| |
| // For AES-192 round keys occupy: K0 .. K7, K10, K11, K8, K9, KLAST |
| VMOV K8.B16, K10.B16 |
| VMOV K9.B16, K11.B16 |
| VMOV KLAST.B16, K8.B16 |
| VLD1.P 16(H0), [K9.B16] |
| VLD1.P 16(H0), [KLAST.B16] |
| TBZ $3, NR, octetsLoop |
| // For AES-256 round keys occupy: K0 .. K7, K10, K11, mem, mem, K8, K9, KLAST |
| VMOV KLAST.B16, K8.B16 |
| VLD1.P 16(H0), [K9.B16] |
| VLD1.P 16(H0), [KLAST.B16] |
| ADD $10*16, ks, H0 |
| MOVD H0, curK |
| |
| octetsLoop: |
| SUB $128, srcPtrLen |
| |
| VMOV CTR.B16, B0.B16 |
| VADD B0.S4, INC.S4, B1.S4 |
| VREV32 B0.B16, B0.B16 |
| VADD B1.S4, INC.S4, B2.S4 |
| VREV32 B1.B16, B1.B16 |
| VADD B2.S4, INC.S4, B3.S4 |
| VREV32 B2.B16, B2.B16 |
| VADD B3.S4, INC.S4, B4.S4 |
| VREV32 B3.B16, B3.B16 |
| VADD B4.S4, INC.S4, B5.S4 |
| VREV32 B4.B16, B4.B16 |
| VADD B5.S4, INC.S4, B6.S4 |
| VREV32 B5.B16, B5.B16 |
| VADD B6.S4, INC.S4, B7.S4 |
| VREV32 B6.B16, B6.B16 |
| VADD B7.S4, INC.S4, CTR.S4 |
| VREV32 B7.B16, B7.B16 |
| |
| aesrndx8(K0) |
| aesrndx8(K1) |
| aesrndx8(K2) |
| aesrndx8(K3) |
| aesrndx8(K4) |
| aesrndx8(K5) |
| aesrndx8(K6) |
| aesrndx8(K7) |
| TBZ $4, NR, octetsFinish |
| aesrndx8(K10) |
| aesrndx8(K11) |
| TBZ $3, NR, octetsFinish |
| VLD1.P 32(curK), [T1.B16, T2.B16] |
| aesrndx8(T1) |
| aesrndx8(T2) |
| MOVD H0, curK |
| octetsFinish: |
| aesrndx8(K8) |
| aesrndlastx8(K9) |
| |
| VEOR KLAST.B16, B0.B16, B0.B16 |
| VEOR KLAST.B16, B1.B16, B1.B16 |
| VEOR KLAST.B16, B2.B16, B2.B16 |
| VEOR KLAST.B16, B3.B16, B3.B16 |
| VEOR KLAST.B16, B4.B16, B4.B16 |
| VEOR KLAST.B16, B5.B16, B5.B16 |
| VEOR KLAST.B16, B6.B16, B6.B16 |
| VEOR KLAST.B16, B7.B16, B7.B16 |
| |
| VLD1.P 32(srcPtr), [T1.B16, T2.B16] |
| VEOR B0.B16, T1.B16, B0.B16 |
| VEOR B1.B16, T2.B16, B1.B16 |
| VST1.P [B0.B16, B1.B16], 32(dstPtr) |
| VLD1.P 32(srcPtr), [T1.B16, T2.B16] |
| VEOR B2.B16, T1.B16, B2.B16 |
| VEOR B3.B16, T2.B16, B3.B16 |
| VST1.P [B2.B16, B3.B16], 32(dstPtr) |
| VLD1.P 32(srcPtr), [T1.B16, T2.B16] |
| VEOR B4.B16, T1.B16, B4.B16 |
| VEOR B5.B16, T2.B16, B5.B16 |
| VST1.P [B4.B16, B5.B16], 32(dstPtr) |
| VLD1.P 32(srcPtr), [T1.B16, T2.B16] |
| VEOR B6.B16, T1.B16, B6.B16 |
| VEOR B7.B16, T2.B16, B7.B16 |
| VST1.P [B6.B16, B7.B16], 32(dstPtr) |
| |
| VLD1.P 32(pTbl), [T1.B16, T2.B16] |
| VREV64 B0.B16, B0.B16 |
| VEOR ACC0.B16, B0.B16, B0.B16 |
| VEXT $8, B0.B16, B0.B16, T0.B16 |
| VEOR B0.B16, T0.B16, T0.B16 |
| VPMULL B0.D1, T1.D1, ACC1.Q1 |
| VPMULL2 B0.D2, T1.D2, ACC0.Q1 |
| VPMULL T0.D1, T2.D1, ACCM.Q1 |
| |
| mulRound(B1) |
| mulRound(B2) |
| mulRound(B3) |
| mulRound(B4) |
| mulRound(B5) |
| mulRound(B6) |
| mulRound(B7) |
| MOVD pTblSave, pTbl |
| reduce() |
| |
| CMP $128, srcPtrLen |
| BGE octetsLoop |
| |
| startSingles: |
| CBZ srcPtrLen, done |
| ADD $14*16, pTbl |
| // Preload H and its Karatsuba precomp |
| VLD1.P (pTbl), [T1.B16, T2.B16] |
| // Preload AES round keys |
| ADD $128, ks |
| VLD1.P 48(ks), [K8.B16, K9.B16, K10.B16] |
| VMOV K10.B16, KLAST.B16 |
| TBZ $4, NR, singlesLoop |
| VLD1.P 32(ks), [B1.B16, B2.B16] |
| VMOV B2.B16, KLAST.B16 |
| TBZ $3, NR, singlesLoop |
| VLD1.P 32(ks), [B3.B16, B4.B16] |
| VMOV B4.B16, KLAST.B16 |
| |
| singlesLoop: |
| CMP $16, srcPtrLen |
| BLT tail |
| SUB $16, srcPtrLen |
| |
| VLD1.P 16(srcPtr), [T0.B16] |
| VEOR KLAST.B16, T0.B16, T0.B16 |
| |
| VREV32 CTR.B16, B0.B16 |
| VADD CTR.S4, INC.S4, CTR.S4 |
| |
| AESE K0.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K1.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K2.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K3.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K4.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K5.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K6.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K7.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K8.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K9.B16, B0.B16 |
| TBZ $4, NR, singlesLast |
| AESMC B0.B16, B0.B16 |
| AESE K10.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE B1.B16, B0.B16 |
| TBZ $3, NR, singlesLast |
| AESMC B0.B16, B0.B16 |
| AESE B2.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE B3.B16, B0.B16 |
| singlesLast: |
| VEOR T0.B16, B0.B16, B0.B16 |
| encReduce: |
| VST1.P [B0.B16], 16(dstPtr) |
| |
| VREV64 B0.B16, B0.B16 |
| VEOR ACC0.B16, B0.B16, B0.B16 |
| |
| VEXT $8, B0.B16, B0.B16, T0.B16 |
| VEOR B0.B16, T0.B16, T0.B16 |
| VPMULL B0.D1, T1.D1, ACC1.Q1 |
| VPMULL2 B0.D2, T1.D2, ACC0.Q1 |
| VPMULL T0.D1, T2.D1, ACCM.Q1 |
| |
| reduce() |
| |
| B singlesLoop |
| tail: |
| CBZ srcPtrLen, done |
| |
| VEOR T0.B16, T0.B16, T0.B16 |
| VEOR T3.B16, T3.B16, T3.B16 |
| MOVD $0, H1 |
| SUB $1, H1 |
| ADD srcPtrLen, srcPtr |
| |
| TBZ $3, srcPtrLen, ld4 |
| MOVD.W -8(srcPtr), H0 |
| VMOV H0, T0.D[0] |
| VMOV H1, T3.D[0] |
| ld4: |
| TBZ $2, srcPtrLen, ld2 |
| MOVW.W -4(srcPtr), H0 |
| VEXT $12, T0.B16, ZERO.B16, T0.B16 |
| VEXT $12, T3.B16, ZERO.B16, T3.B16 |
| VMOV H0, T0.S[0] |
| VMOV H1, T3.S[0] |
| ld2: |
| TBZ $1, srcPtrLen, ld1 |
| MOVH.W -2(srcPtr), H0 |
| VEXT $14, T0.B16, ZERO.B16, T0.B16 |
| VEXT $14, T3.B16, ZERO.B16, T3.B16 |
| VMOV H0, T0.H[0] |
| VMOV H1, T3.H[0] |
| ld1: |
| TBZ $0, srcPtrLen, ld0 |
| MOVB.W -1(srcPtr), H0 |
| VEXT $15, T0.B16, ZERO.B16, T0.B16 |
| VEXT $15, T3.B16, ZERO.B16, T3.B16 |
| VMOV H0, T0.B[0] |
| VMOV H1, T3.B[0] |
| ld0: |
| |
| MOVD ZR, srcPtrLen |
| VEOR KLAST.B16, T0.B16, T0.B16 |
| VREV32 CTR.B16, B0.B16 |
| |
| AESE K0.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K1.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K2.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K3.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K4.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K5.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K6.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K7.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K8.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K9.B16, B0.B16 |
| TBZ $4, NR, tailLast |
| AESMC B0.B16, B0.B16 |
| AESE K10.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE B1.B16, B0.B16 |
| TBZ $3, NR, tailLast |
| AESMC B0.B16, B0.B16 |
| AESE B2.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE B3.B16, B0.B16 |
| |
| tailLast: |
| VEOR T0.B16, B0.B16, B0.B16 |
| VAND T3.B16, B0.B16, B0.B16 |
| B encReduce |
| |
| done: |
| VST1 [ACC0.B16], (tPtr) |
| RET |
| |
| // func gcmAesDec(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32) |
| TEXT ·gcmAesDec(SB),NOSPLIT,$0 |
| MOVD productTable+0(FP), pTbl |
| MOVD dst+8(FP), dstPtr |
| MOVD src_base+32(FP), srcPtr |
| MOVD src_len+40(FP), srcPtrLen |
| MOVD ctr+56(FP), ctrPtr |
| MOVD T+64(FP), tPtr |
| MOVD ks_base+72(FP), ks |
| MOVD ks_len+80(FP), NR |
| |
| MOVD $0xC2, H1 |
| LSL $56, H1 |
| MOVD $1, H0 |
| VMOV H1, POLY.D[0] |
| VMOV H0, POLY.D[1] |
| VEOR ZERO.B16, ZERO.B16, ZERO.B16 |
| // Compute NR from len(ks) |
| MOVD pTbl, pTblSave |
| // Current tag, after AAD |
| VLD1 (tPtr), [ACC0.B16] |
| VEOR ACC1.B16, ACC1.B16, ACC1.B16 |
| VEOR ACCM.B16, ACCM.B16, ACCM.B16 |
| // Prepare initial counter, and the increment vector |
| VLD1 (ctrPtr), [CTR.B16] |
| VEOR INC.B16, INC.B16, INC.B16 |
| MOVD $1, H0 |
| VMOV H0, INC.S[3] |
| VREV32 CTR.B16, CTR.B16 |
| VADD CTR.S4, INC.S4, CTR.S4 |
| |
| MOVD ks, H0 |
| // For AES-128 round keys are stored in: K0 .. K10, KLAST |
| VLD1.P 64(H0), [K0.B16, K1.B16, K2.B16, K3.B16] |
| VLD1.P 64(H0), [K4.B16, K5.B16, K6.B16, K7.B16] |
| VLD1.P 48(H0), [K8.B16, K9.B16, K10.B16] |
| VMOV K10.B16, KLAST.B16 |
| |
| // Skip to <8 blocks loop |
| CMP $128, srcPtrLen |
| BLT startSingles |
| // There are at least 8 blocks to encrypt |
| TBZ $4, NR, octetsLoop |
| |
| // For AES-192 round keys occupy: K0 .. K7, K10, K11, K8, K9, KLAST |
| VMOV K8.B16, K10.B16 |
| VMOV K9.B16, K11.B16 |
| VMOV KLAST.B16, K8.B16 |
| VLD1.P 16(H0), [K9.B16] |
| VLD1.P 16(H0), [KLAST.B16] |
| TBZ $3, NR, octetsLoop |
| // For AES-256 round keys occupy: K0 .. K7, K10, K11, mem, mem, K8, K9, KLAST |
| VMOV KLAST.B16, K8.B16 |
| VLD1.P 16(H0), [K9.B16] |
| VLD1.P 16(H0), [KLAST.B16] |
| ADD $10*16, ks, H0 |
| MOVD H0, curK |
| |
| octetsLoop: |
| SUB $128, srcPtrLen |
| |
| VMOV CTR.B16, B0.B16 |
| VADD B0.S4, INC.S4, B1.S4 |
| VREV32 B0.B16, B0.B16 |
| VADD B1.S4, INC.S4, B2.S4 |
| VREV32 B1.B16, B1.B16 |
| VADD B2.S4, INC.S4, B3.S4 |
| VREV32 B2.B16, B2.B16 |
| VADD B3.S4, INC.S4, B4.S4 |
| VREV32 B3.B16, B3.B16 |
| VADD B4.S4, INC.S4, B5.S4 |
| VREV32 B4.B16, B4.B16 |
| VADD B5.S4, INC.S4, B6.S4 |
| VREV32 B5.B16, B5.B16 |
| VADD B6.S4, INC.S4, B7.S4 |
| VREV32 B6.B16, B6.B16 |
| VADD B7.S4, INC.S4, CTR.S4 |
| VREV32 B7.B16, B7.B16 |
| |
| aesrndx8(K0) |
| aesrndx8(K1) |
| aesrndx8(K2) |
| aesrndx8(K3) |
| aesrndx8(K4) |
| aesrndx8(K5) |
| aesrndx8(K6) |
| aesrndx8(K7) |
| TBZ $4, NR, octetsFinish |
| aesrndx8(K10) |
| aesrndx8(K11) |
| TBZ $3, NR, octetsFinish |
| VLD1.P 32(curK), [T1.B16, T2.B16] |
| aesrndx8(T1) |
| aesrndx8(T2) |
| MOVD H0, curK |
| octetsFinish: |
| aesrndx8(K8) |
| aesrndlastx8(K9) |
| |
| VEOR KLAST.B16, B0.B16, T1.B16 |
| VEOR KLAST.B16, B1.B16, T2.B16 |
| VEOR KLAST.B16, B2.B16, B2.B16 |
| VEOR KLAST.B16, B3.B16, B3.B16 |
| VEOR KLAST.B16, B4.B16, B4.B16 |
| VEOR KLAST.B16, B5.B16, B5.B16 |
| VEOR KLAST.B16, B6.B16, B6.B16 |
| VEOR KLAST.B16, B7.B16, B7.B16 |
| |
| VLD1.P 32(srcPtr), [B0.B16, B1.B16] |
| VEOR B0.B16, T1.B16, T1.B16 |
| VEOR B1.B16, T2.B16, T2.B16 |
| VST1.P [T1.B16, T2.B16], 32(dstPtr) |
| |
| VLD1.P 32(pTbl), [T1.B16, T2.B16] |
| VREV64 B0.B16, B0.B16 |
| VEOR ACC0.B16, B0.B16, B0.B16 |
| VEXT $8, B0.B16, B0.B16, T0.B16 |
| VEOR B0.B16, T0.B16, T0.B16 |
| VPMULL B0.D1, T1.D1, ACC1.Q1 |
| VPMULL2 B0.D2, T1.D2, ACC0.Q1 |
| VPMULL T0.D1, T2.D1, ACCM.Q1 |
| mulRound(B1) |
| |
| VLD1.P 32(srcPtr), [B0.B16, B1.B16] |
| VEOR B2.B16, B0.B16, T1.B16 |
| VEOR B3.B16, B1.B16, T2.B16 |
| VST1.P [T1.B16, T2.B16], 32(dstPtr) |
| mulRound(B0) |
| mulRound(B1) |
| |
| VLD1.P 32(srcPtr), [B0.B16, B1.B16] |
| VEOR B4.B16, B0.B16, T1.B16 |
| VEOR B5.B16, B1.B16, T2.B16 |
| VST1.P [T1.B16, T2.B16], 32(dstPtr) |
| mulRound(B0) |
| mulRound(B1) |
| |
| VLD1.P 32(srcPtr), [B0.B16, B1.B16] |
| VEOR B6.B16, B0.B16, T1.B16 |
| VEOR B7.B16, B1.B16, T2.B16 |
| VST1.P [T1.B16, T2.B16], 32(dstPtr) |
| mulRound(B0) |
| mulRound(B1) |
| |
| MOVD pTblSave, pTbl |
| reduce() |
| |
| CMP $128, srcPtrLen |
| BGE octetsLoop |
| |
| startSingles: |
| CBZ srcPtrLen, done |
| ADD $14*16, pTbl |
| // Preload H and its Karatsuba precomp |
| VLD1.P (pTbl), [T1.B16, T2.B16] |
| // Preload AES round keys |
| ADD $128, ks |
| VLD1.P 48(ks), [K8.B16, K9.B16, K10.B16] |
| VMOV K10.B16, KLAST.B16 |
| TBZ $4, NR, singlesLoop |
| VLD1.P 32(ks), [B1.B16, B2.B16] |
| VMOV B2.B16, KLAST.B16 |
| TBZ $3, NR, singlesLoop |
| VLD1.P 32(ks), [B3.B16, B4.B16] |
| VMOV B4.B16, KLAST.B16 |
| |
| singlesLoop: |
| CMP $16, srcPtrLen |
| BLT tail |
| SUB $16, srcPtrLen |
| |
| VLD1.P 16(srcPtr), [T0.B16] |
| VREV64 T0.B16, B5.B16 |
| VEOR KLAST.B16, T0.B16, T0.B16 |
| |
| VREV32 CTR.B16, B0.B16 |
| VADD CTR.S4, INC.S4, CTR.S4 |
| |
| AESE K0.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K1.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K2.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K3.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K4.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K5.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K6.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K7.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K8.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K9.B16, B0.B16 |
| TBZ $4, NR, singlesLast |
| AESMC B0.B16, B0.B16 |
| AESE K10.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE B1.B16, B0.B16 |
| TBZ $3, NR, singlesLast |
| AESMC B0.B16, B0.B16 |
| AESE B2.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE B3.B16, B0.B16 |
| singlesLast: |
| VEOR T0.B16, B0.B16, B0.B16 |
| |
| VST1.P [B0.B16], 16(dstPtr) |
| |
| VEOR ACC0.B16, B5.B16, B5.B16 |
| VEXT $8, B5.B16, B5.B16, T0.B16 |
| VEOR B5.B16, T0.B16, T0.B16 |
| VPMULL B5.D1, T1.D1, ACC1.Q1 |
| VPMULL2 B5.D2, T1.D2, ACC0.Q1 |
| VPMULL T0.D1, T2.D1, ACCM.Q1 |
| reduce() |
| |
| B singlesLoop |
| tail: |
| CBZ srcPtrLen, done |
| |
| VREV32 CTR.B16, B0.B16 |
| VADD CTR.S4, INC.S4, CTR.S4 |
| |
| AESE K0.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K1.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K2.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K3.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K4.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K5.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K6.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K7.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K8.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE K9.B16, B0.B16 |
| TBZ $4, NR, tailLast |
| AESMC B0.B16, B0.B16 |
| AESE K10.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE B1.B16, B0.B16 |
| TBZ $3, NR, tailLast |
| AESMC B0.B16, B0.B16 |
| AESE B2.B16, B0.B16 |
| AESMC B0.B16, B0.B16 |
| AESE B3.B16, B0.B16 |
| tailLast: |
| VEOR KLAST.B16, B0.B16, B0.B16 |
| |
| // Assuming it is safe to load past dstPtr due to the presence of the tag |
| VLD1 (srcPtr), [B5.B16] |
| |
| VEOR B5.B16, B0.B16, B0.B16 |
| |
| VEOR T3.B16, T3.B16, T3.B16 |
| MOVD $0, H1 |
| SUB $1, H1 |
| |
| TBZ $3, srcPtrLen, ld4 |
| VMOV B0.D[0], H0 |
| MOVD.P H0, 8(dstPtr) |
| VMOV H1, T3.D[0] |
| VEXT $8, ZERO.B16, B0.B16, B0.B16 |
| ld4: |
| TBZ $2, srcPtrLen, ld2 |
| VMOV B0.S[0], H0 |
| MOVW.P H0, 4(dstPtr) |
| VEXT $12, T3.B16, ZERO.B16, T3.B16 |
| VMOV H1, T3.S[0] |
| VEXT $4, ZERO.B16, B0.B16, B0.B16 |
| ld2: |
| TBZ $1, srcPtrLen, ld1 |
| VMOV B0.H[0], H0 |
| MOVH.P H0, 2(dstPtr) |
| VEXT $14, T3.B16, ZERO.B16, T3.B16 |
| VMOV H1, T3.H[0] |
| VEXT $2, ZERO.B16, B0.B16, B0.B16 |
| ld1: |
| TBZ $0, srcPtrLen, ld0 |
| VMOV B0.B[0], H0 |
| MOVB.P H0, 1(dstPtr) |
| VEXT $15, T3.B16, ZERO.B16, T3.B16 |
| VMOV H1, T3.B[0] |
| ld0: |
| |
| VAND T3.B16, B5.B16, B5.B16 |
| VREV64 B5.B16, B5.B16 |
| |
| VEOR ACC0.B16, B5.B16, B5.B16 |
| VEXT $8, B5.B16, B5.B16, T0.B16 |
| VEOR B5.B16, T0.B16, T0.B16 |
| VPMULL B5.D1, T1.D1, ACC1.Q1 |
| VPMULL2 B5.D2, T1.D2, ACC0.Q1 |
| VPMULL T0.D1, T2.D1, ACCM.Q1 |
| reduce() |
| done: |
| VST1 [ACC0.B16], (tPtr) |
| |
| RET |
