| // Code generated by command: go run chacha20poly1305_amd64_asm.go -out ../chacha20poly1305_amd64.s -pkg chacha20poly1305. DO NOT EDIT. |
| |
| //go:build gc && !purego |
| |
| #include "textflag.h" |
| |
| // func polyHashADInternal<>() |
| TEXT polyHashADInternal<>(SB), NOSPLIT, $0 |
| XORQ R10, R10 |
| XORQ R11, R11 |
| XORQ R12, R12 |
| CMPQ R9, $0x0d |
| JNE hashADLoop |
| MOVQ (CX), R10 |
| MOVQ 5(CX), R11 |
| SHRQ $0x18, R11 |
| MOVQ $0x00000001, R12 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| RET |
| |
| hashADLoop: |
| // Hash in 16 byte chunks |
| CMPQ R9, $0x10 |
| JB hashADTail |
| ADDQ (CX), R10 |
| ADCQ 8(CX), R11 |
| ADCQ $0x01, R12 |
| LEAQ 16(CX), CX |
| SUBQ $0x10, R9 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| JMP hashADLoop |
| |
| hashADTail: |
| CMPQ R9, $0x00 |
| JE hashADDone |
| |
| // Hash last < 16 byte tail |
| XORQ R13, R13 |
| XORQ R14, R14 |
| XORQ R15, R15 |
| ADDQ R9, CX |
| |
| hashADTailLoop: |
| SHLQ $0x08, R13, R14 |
| SHLQ $0x08, R13 |
| MOVB -1(CX), R15 |
| XORQ R15, R13 |
| DECQ CX |
| DECQ R9 |
| JNE hashADTailLoop |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| |
| hashADDone: |
| RET |
| |
| // func chacha20Poly1305Open(dst []byte, key []uint32, src []byte, ad []byte) bool |
| // Requires: AVX, AVX2, BMI2, CMOV, SSE2 |
| TEXT ·chacha20Poly1305Open(SB), $288-97 |
| // For aligned stack access |
| MOVQ SP, BP |
| ADDQ $0x20, BP |
| ANDQ $-32, BP |
| MOVQ dst_base+0(FP), DI |
| MOVQ key_base+24(FP), R8 |
| MOVQ src_base+48(FP), SI |
| MOVQ src_len+56(FP), BX |
| MOVQ ad_base+72(FP), CX |
| VZEROUPPER |
| VMOVDQU ·chacha20Constants<>+0(SB), Y0 |
| VBROADCASTI128 16(R8), Y14 |
| VBROADCASTI128 32(R8), Y12 |
| VBROADCASTI128 48(R8), Y4 |
| VPADDD ·avx2InitMask<>+0(SB), Y4, Y4 |
| |
| // Special optimization, for very short buffers |
| CMPQ BX, $0xc0 |
| JBE openAVX2192 |
| CMPQ BX, $0x00000140 |
| JBE openAVX2320 |
| |
| // For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream |
| VMOVDQA Y14, 32(BP) |
| VMOVDQA Y12, 64(BP) |
| VMOVDQA Y4, 192(BP) |
| MOVQ $0x0000000a, R9 |
| |
| openAVX2PreparePolyKey: |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPALIGNR $0x04, Y14, Y14, Y14 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x0c, Y4, Y4, Y4 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPALIGNR $0x0c, Y14, Y14, Y14 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x04, Y4, Y4, Y4 |
| DECQ R9 |
| JNE openAVX2PreparePolyKey |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 |
| VPADDD 32(BP), Y14, Y14 |
| VPADDD 64(BP), Y12, Y12 |
| VPADDD 192(BP), Y4, Y4 |
| VPERM2I128 $0x02, Y0, Y14, Y3 |
| |
| // Clamp and store poly key |
| VPAND ·polyClampMask<>+0(SB), Y3, Y3 |
| VMOVDQA Y3, (BP) |
| |
| // Stream for the first 64 bytes |
| VPERM2I128 $0x13, Y0, Y14, Y0 |
| VPERM2I128 $0x13, Y12, Y4, Y14 |
| |
| // Hash AD + first 64 bytes |
| MOVQ ad_len+80(FP), R9 |
| CALL polyHashADInternal<>(SB) |
| XORQ CX, CX |
| |
| openAVX2InitialHash64: |
| ADDQ (SI)(CX*1), R10 |
| ADCQ 8(SI)(CX*1), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| ADDQ $0x10, CX |
| CMPQ CX, $0x40 |
| JNE openAVX2InitialHash64 |
| |
| // Decrypt the first 64 bytes |
| VPXOR (SI), Y0, Y0 |
| VPXOR 32(SI), Y14, Y14 |
| VMOVDQU Y0, (DI) |
| VMOVDQU Y14, 32(DI) |
| LEAQ 64(SI), SI |
| LEAQ 64(DI), DI |
| SUBQ $0x40, BX |
| |
| openAVX2MainLoop: |
| CMPQ BX, $0x00000200 |
| JB openAVX2MainLoopDone |
| |
| // Load state, increment counter blocks, store the incremented counters |
| VMOVDQU ·chacha20Constants<>+0(SB), Y0 |
| VMOVDQA Y0, Y5 |
| VMOVDQA Y0, Y6 |
| VMOVDQA Y0, Y7 |
| VMOVDQA 32(BP), Y14 |
| VMOVDQA Y14, Y9 |
| VMOVDQA Y14, Y10 |
| VMOVDQA Y14, Y11 |
| VMOVDQA 64(BP), Y12 |
| VMOVDQA Y12, Y13 |
| VMOVDQA Y12, Y8 |
| VMOVDQA Y12, Y15 |
| VMOVDQA 192(BP), Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 |
| VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 |
| VMOVDQA Y4, 96(BP) |
| VMOVDQA Y1, 128(BP) |
| VMOVDQA Y2, 160(BP) |
| VMOVDQA Y3, 192(BP) |
| XORQ CX, CX |
| |
| openAVX2InternalLoop: |
| ADDQ (SI)(CX*1), R10 |
| ADCQ 8(SI)(CX*1), R11 |
| ADCQ $0x01, R12 |
| VPADDD Y14, Y0, Y0 |
| VPADDD Y9, Y5, Y5 |
| VPADDD Y10, Y6, Y6 |
| VPADDD Y11, Y7, Y7 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| VPXOR Y0, Y4, Y4 |
| VPXOR Y5, Y1, Y1 |
| VPXOR Y6, Y2, Y2 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| VPADDD Y4, Y12, Y12 |
| VPADDD Y1, Y13, Y13 |
| VPADDD Y2, Y8, Y8 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y12, Y14, Y14 |
| VPXOR Y13, Y9, Y9 |
| VPXOR Y8, Y10, Y10 |
| VPXOR Y15, Y11, Y11 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| VMOVDQA Y15, 224(BP) |
| VPSLLD $0x0c, Y14, Y15 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPSLLD $0x0c, Y9, Y15 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPSLLD $0x0c, Y10, Y15 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPSLLD $0x0c, Y11, Y15 |
| VPSRLD $0x14, Y11, Y11 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA 224(BP), Y15 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| VPADDD Y14, Y0, Y0 |
| VPADDD Y9, Y5, Y5 |
| VPADDD Y10, Y6, Y6 |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y0, Y4, Y4 |
| VPXOR Y5, Y1, Y1 |
| VPXOR Y6, Y2, Y2 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 |
| ADDQ 16(SI)(CX*1), R10 |
| ADCQ 24(SI)(CX*1), R11 |
| ADCQ $0x01, R12 |
| VPADDD Y4, Y12, Y12 |
| VPADDD Y1, Y13, Y13 |
| VPADDD Y2, Y8, Y8 |
| VPADDD Y3, Y15, Y15 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| VPXOR Y12, Y14, Y14 |
| VPXOR Y13, Y9, Y9 |
| VPXOR Y8, Y10, Y10 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA Y15, 224(BP) |
| VPSLLD $0x07, Y14, Y15 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPSLLD $0x07, Y9, Y15 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPSLLD $0x07, Y10, Y15 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPSLLD $0x07, Y11, Y15 |
| VPSRLD $0x19, Y11, Y11 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA 224(BP), Y15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| VPALIGNR $0x04, Y14, Y14, Y14 |
| VPALIGNR $0x04, Y9, Y9, Y9 |
| VPALIGNR $0x04, Y10, Y10, Y10 |
| VPALIGNR $0x04, Y11, Y11, Y11 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x08, Y15, Y15, Y15 |
| VPALIGNR $0x0c, Y4, Y4, Y4 |
| VPALIGNR $0x0c, Y1, Y1, Y1 |
| VPALIGNR $0x0c, Y2, Y2, Y2 |
| VPALIGNR $0x0c, Y3, Y3, Y3 |
| VPADDD Y14, Y0, Y0 |
| VPADDD Y9, Y5, Y5 |
| VPADDD Y10, Y6, Y6 |
| VPADDD Y11, Y7, Y7 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| VPXOR Y0, Y4, Y4 |
| VPXOR Y5, Y1, Y1 |
| VPXOR Y6, Y2, Y2 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| VPADDD Y4, Y12, Y12 |
| VPADDD Y1, Y13, Y13 |
| VPADDD Y2, Y8, Y8 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y12, Y14, Y14 |
| VPXOR Y13, Y9, Y9 |
| VPXOR Y8, Y10, Y10 |
| VPXOR Y15, Y11, Y11 |
| ADDQ 32(SI)(CX*1), R10 |
| ADCQ 40(SI)(CX*1), R11 |
| ADCQ $0x01, R12 |
| LEAQ 48(CX), CX |
| VMOVDQA Y15, 224(BP) |
| VPSLLD $0x0c, Y14, Y15 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPSLLD $0x0c, Y9, Y15 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPSLLD $0x0c, Y10, Y15 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPSLLD $0x0c, Y11, Y15 |
| VPSRLD $0x14, Y11, Y11 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA 224(BP), Y15 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| VPADDD Y14, Y0, Y0 |
| VPADDD Y9, Y5, Y5 |
| VPADDD Y10, Y6, Y6 |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y0, Y4, Y4 |
| VPXOR Y5, Y1, Y1 |
| VPXOR Y6, Y2, Y2 |
| VPXOR Y7, Y3, Y3 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 |
| VPADDD Y4, Y12, Y12 |
| VPADDD Y1, Y13, Y13 |
| VPADDD Y2, Y8, Y8 |
| VPADDD Y3, Y15, Y15 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| VPXOR Y12, Y14, Y14 |
| VPXOR Y13, Y9, Y9 |
| VPXOR Y8, Y10, Y10 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA Y15, 224(BP) |
| VPSLLD $0x07, Y14, Y15 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPSLLD $0x07, Y9, Y15 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPSLLD $0x07, Y10, Y15 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPSLLD $0x07, Y11, Y15 |
| VPSRLD $0x19, Y11, Y11 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA 224(BP), Y15 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| VPALIGNR $0x0c, Y14, Y14, Y14 |
| VPALIGNR $0x0c, Y9, Y9, Y9 |
| VPALIGNR $0x0c, Y10, Y10, Y10 |
| VPALIGNR $0x0c, Y11, Y11, Y11 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x08, Y15, Y15, Y15 |
| VPALIGNR $0x04, Y4, Y4, Y4 |
| VPALIGNR $0x04, Y1, Y1, Y1 |
| VPALIGNR $0x04, Y2, Y2, Y2 |
| VPALIGNR $0x04, Y3, Y3, Y3 |
| CMPQ CX, $0x000001e0 |
| JNE openAVX2InternalLoop |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 |
| VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 |
| VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 |
| VPADDD 32(BP), Y14, Y14 |
| VPADDD 32(BP), Y9, Y9 |
| VPADDD 32(BP), Y10, Y10 |
| VPADDD 32(BP), Y11, Y11 |
| VPADDD 64(BP), Y12, Y12 |
| VPADDD 64(BP), Y13, Y13 |
| VPADDD 64(BP), Y8, Y8 |
| VPADDD 64(BP), Y15, Y15 |
| VPADDD 96(BP), Y4, Y4 |
| VPADDD 128(BP), Y1, Y1 |
| VPADDD 160(BP), Y2, Y2 |
| VPADDD 192(BP), Y3, Y3 |
| VMOVDQA Y15, 224(BP) |
| |
| // We only hashed 480 of the 512 bytes available - hash the remaining 32 here |
| ADDQ 480(SI), R10 |
| ADCQ 488(SI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| VPERM2I128 $0x02, Y0, Y14, Y15 |
| VPERM2I128 $0x13, Y0, Y14, Y14 |
| VPERM2I128 $0x02, Y12, Y4, Y0 |
| VPERM2I128 $0x13, Y12, Y4, Y12 |
| VPXOR (SI), Y15, Y15 |
| VPXOR 32(SI), Y0, Y0 |
| VPXOR 64(SI), Y14, Y14 |
| VPXOR 96(SI), Y12, Y12 |
| VMOVDQU Y15, (DI) |
| VMOVDQU Y0, 32(DI) |
| VMOVDQU Y14, 64(DI) |
| VMOVDQU Y12, 96(DI) |
| VPERM2I128 $0x02, Y5, Y9, Y0 |
| VPERM2I128 $0x02, Y13, Y1, Y14 |
| VPERM2I128 $0x13, Y5, Y9, Y12 |
| VPERM2I128 $0x13, Y13, Y1, Y4 |
| VPXOR 128(SI), Y0, Y0 |
| VPXOR 160(SI), Y14, Y14 |
| VPXOR 192(SI), Y12, Y12 |
| VPXOR 224(SI), Y4, Y4 |
| VMOVDQU Y0, 128(DI) |
| VMOVDQU Y14, 160(DI) |
| VMOVDQU Y12, 192(DI) |
| VMOVDQU Y4, 224(DI) |
| |
| // and here |
| ADDQ 496(SI), R10 |
| ADCQ 504(SI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| VPERM2I128 $0x02, Y6, Y10, Y0 |
| VPERM2I128 $0x02, Y8, Y2, Y14 |
| VPERM2I128 $0x13, Y6, Y10, Y12 |
| VPERM2I128 $0x13, Y8, Y2, Y4 |
| VPXOR 256(SI), Y0, Y0 |
| VPXOR 288(SI), Y14, Y14 |
| VPXOR 320(SI), Y12, Y12 |
| VPXOR 352(SI), Y4, Y4 |
| VMOVDQU Y0, 256(DI) |
| VMOVDQU Y14, 288(DI) |
| VMOVDQU Y12, 320(DI) |
| VMOVDQU Y4, 352(DI) |
| VPERM2I128 $0x02, Y7, Y11, Y0 |
| VPERM2I128 $0x02, 224(BP), Y3, Y14 |
| VPERM2I128 $0x13, Y7, Y11, Y12 |
| VPERM2I128 $0x13, 224(BP), Y3, Y4 |
| VPXOR 384(SI), Y0, Y0 |
| VPXOR 416(SI), Y14, Y14 |
| VPXOR 448(SI), Y12, Y12 |
| VPXOR 480(SI), Y4, Y4 |
| VMOVDQU Y0, 384(DI) |
| VMOVDQU Y14, 416(DI) |
| VMOVDQU Y12, 448(DI) |
| VMOVDQU Y4, 480(DI) |
| LEAQ 512(SI), SI |
| LEAQ 512(DI), DI |
| SUBQ $0x00000200, BX |
| JMP openAVX2MainLoop |
| |
| openAVX2MainLoopDone: |
| // Handle the various tail sizes efficiently |
| TESTQ BX, BX |
| JE openSSEFinalize |
| CMPQ BX, $0x80 |
| JBE openAVX2Tail128 |
| CMPQ BX, $0x00000100 |
| JBE openAVX2Tail256 |
| CMPQ BX, $0x00000180 |
| JBE openAVX2Tail384 |
| JMP openAVX2Tail512 |
| |
| openSSEFinalize: |
| // Hash in the PT, AAD lengths |
| ADDQ ad_len+80(FP), R10 |
| ADCQ src_len+56(FP), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| |
| // Final reduce |
| MOVQ R10, R13 |
| MOVQ R11, R14 |
| MOVQ R12, R15 |
| SUBQ $-5, R10 |
| SBBQ $-1, R11 |
| SBBQ $0x03, R12 |
| CMOVQCS R13, R10 |
| CMOVQCS R14, R11 |
| CMOVQCS R15, R12 |
| |
| // Add in the "s" part of the key |
| ADDQ 16(BP), R10 |
| ADCQ 24(BP), R11 |
| |
| // Finally, constant time compare to the tag at the end of the message |
| XORQ AX, AX |
| MOVQ $0x00000001, DX |
| XORQ (SI), R10 |
| XORQ 8(SI), R11 |
| ORQ R11, R10 |
| CMOVQEQ DX, AX |
| |
| // Return true iff tags are equal |
| MOVB AX, ret+96(FP) |
| RET |
| |
| openSSETail16: |
| TESTQ BX, BX |
| JE openSSEFinalize |
| |
| // We can safely load the CT from the end, because it is padded with the MAC |
| MOVQ BX, R9 |
| SHLQ $0x04, R9 |
| LEAQ ·andMask<>+0(SB), R13 |
| MOVOU (SI), X12 |
| ADDQ BX, SI |
| PAND -16(R13)(R9*1), X12 |
| MOVO X12, 64(BP) |
| MOVQ X12, R13 |
| MOVQ 72(BP), R14 |
| PXOR X1, X12 |
| |
| // We can only store one byte at a time, since plaintext can be shorter than 16 bytes |
| openSSETail16Store: |
| MOVQ X12, R8 |
| MOVB R8, (DI) |
| PSRLDQ $0x01, X12 |
| INCQ DI |
| DECQ BX |
| JNE openSSETail16Store |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| JMP openSSEFinalize |
| |
| openAVX2192: |
| VMOVDQA Y0, Y5 |
| VMOVDQA Y14, Y9 |
| VMOVDQA Y12, Y13 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 |
| VMOVDQA Y0, Y6 |
| VMOVDQA Y14, Y10 |
| VMOVDQA Y12, Y8 |
| VMOVDQA Y4, Y2 |
| VMOVDQA Y1, Y15 |
| MOVQ $0x0000000a, R9 |
| |
| openAVX2192InnerCipherLoop: |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPALIGNR $0x04, Y14, Y14, Y14 |
| VPALIGNR $0x04, Y9, Y9, Y9 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x0c, Y4, Y4, Y4 |
| VPALIGNR $0x0c, Y1, Y1, Y1 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPALIGNR $0x0c, Y14, Y14, Y14 |
| VPALIGNR $0x0c, Y9, Y9, Y9 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x04, Y4, Y4, Y4 |
| VPALIGNR $0x04, Y1, Y1, Y1 |
| DECQ R9 |
| JNE openAVX2192InnerCipherLoop |
| VPADDD Y6, Y0, Y0 |
| VPADDD Y6, Y5, Y5 |
| VPADDD Y10, Y14, Y14 |
| VPADDD Y10, Y9, Y9 |
| VPADDD Y8, Y12, Y12 |
| VPADDD Y8, Y13, Y13 |
| VPADDD Y2, Y4, Y4 |
| VPADDD Y15, Y1, Y1 |
| VPERM2I128 $0x02, Y0, Y14, Y3 |
| |
| // Clamp and store poly key |
| VPAND ·polyClampMask<>+0(SB), Y3, Y3 |
| VMOVDQA Y3, (BP) |
| |
| // Stream for up to 192 bytes |
| VPERM2I128 $0x13, Y0, Y14, Y0 |
| VPERM2I128 $0x13, Y12, Y4, Y14 |
| VPERM2I128 $0x02, Y5, Y9, Y12 |
| VPERM2I128 $0x02, Y13, Y1, Y4 |
| VPERM2I128 $0x13, Y5, Y9, Y5 |
| VPERM2I128 $0x13, Y13, Y1, Y9 |
| |
| openAVX2ShortOpen: |
| // Hash |
| MOVQ ad_len+80(FP), R9 |
| CALL polyHashADInternal<>(SB) |
| |
| openAVX2ShortOpenLoop: |
| CMPQ BX, $0x20 |
| JB openAVX2ShortTail32 |
| SUBQ $0x20, BX |
| |
| // Load for hashing |
| ADDQ (SI), R10 |
| ADCQ 8(SI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| ADDQ 16(SI), R10 |
| ADCQ 24(SI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| |
| // Load for decryption |
| VPXOR (SI), Y0, Y0 |
| VMOVDQU Y0, (DI) |
| LEAQ 32(SI), SI |
| LEAQ 32(DI), DI |
| |
| // Shift stream left |
| VMOVDQA Y14, Y0 |
| VMOVDQA Y12, Y14 |
| VMOVDQA Y4, Y12 |
| VMOVDQA Y5, Y4 |
| VMOVDQA Y9, Y5 |
| VMOVDQA Y13, Y9 |
| VMOVDQA Y1, Y13 |
| VMOVDQA Y6, Y1 |
| VMOVDQA Y10, Y6 |
| JMP openAVX2ShortOpenLoop |
| |
| openAVX2ShortTail32: |
| CMPQ BX, $0x10 |
| VMOVDQA X0, X1 |
| JB openAVX2ShortDone |
| SUBQ $0x10, BX |
| |
| // Load for hashing |
| ADDQ (SI), R10 |
| ADCQ 8(SI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| |
| // Load for decryption |
| VPXOR (SI), X0, X12 |
| VMOVDQU X12, (DI) |
| LEAQ 16(SI), SI |
| LEAQ 16(DI), DI |
| VPERM2I128 $0x11, Y0, Y0, Y0 |
| VMOVDQA X0, X1 |
| |
| openAVX2ShortDone: |
| VZEROUPPER |
| JMP openSSETail16 |
| |
| openAVX2320: |
| VMOVDQA Y0, Y5 |
| VMOVDQA Y14, Y9 |
| VMOVDQA Y12, Y13 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 |
| VMOVDQA Y0, Y6 |
| VMOVDQA Y14, Y10 |
| VMOVDQA Y12, Y8 |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 |
| VMOVDQA Y14, Y7 |
| VMOVDQA Y12, Y11 |
| VMOVDQA Y4, Y15 |
| MOVQ $0x0000000a, R9 |
| |
| openAVX2320InnerCipherLoop: |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x0c, Y10, Y3 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y3, Y10, Y10 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x07, Y10, Y3 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y3, Y10, Y10 |
| VPALIGNR $0x04, Y14, Y14, Y14 |
| VPALIGNR $0x04, Y9, Y9, Y9 |
| VPALIGNR $0x04, Y10, Y10, Y10 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x0c, Y4, Y4, Y4 |
| VPALIGNR $0x0c, Y1, Y1, Y1 |
| VPALIGNR $0x0c, Y2, Y2, Y2 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x0c, Y10, Y3 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y3, Y10, Y10 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x07, Y10, Y3 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y3, Y10, Y10 |
| VPALIGNR $0x0c, Y14, Y14, Y14 |
| VPALIGNR $0x0c, Y9, Y9, Y9 |
| VPALIGNR $0x0c, Y10, Y10, Y10 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x04, Y4, Y4, Y4 |
| VPALIGNR $0x04, Y1, Y1, Y1 |
| VPALIGNR $0x04, Y2, Y2, Y2 |
| DECQ R9 |
| JNE openAVX2320InnerCipherLoop |
| VMOVDQA ·chacha20Constants<>+0(SB), Y3 |
| VPADDD Y3, Y0, Y0 |
| VPADDD Y3, Y5, Y5 |
| VPADDD Y3, Y6, Y6 |
| VPADDD Y7, Y14, Y14 |
| VPADDD Y7, Y9, Y9 |
| VPADDD Y7, Y10, Y10 |
| VPADDD Y11, Y12, Y12 |
| VPADDD Y11, Y13, Y13 |
| VPADDD Y11, Y8, Y8 |
| VMOVDQA ·avx2IncMask<>+0(SB), Y3 |
| VPADDD Y15, Y4, Y4 |
| VPADDD Y3, Y15, Y15 |
| VPADDD Y15, Y1, Y1 |
| VPADDD Y3, Y15, Y15 |
| VPADDD Y15, Y2, Y2 |
| |
| // Clamp and store poly key |
| VPERM2I128 $0x02, Y0, Y14, Y3 |
| VPAND ·polyClampMask<>+0(SB), Y3, Y3 |
| VMOVDQA Y3, (BP) |
| |
| // Stream for up to 320 bytes |
| VPERM2I128 $0x13, Y0, Y14, Y0 |
| VPERM2I128 $0x13, Y12, Y4, Y14 |
| VPERM2I128 $0x02, Y5, Y9, Y12 |
| VPERM2I128 $0x02, Y13, Y1, Y4 |
| VPERM2I128 $0x13, Y5, Y9, Y5 |
| VPERM2I128 $0x13, Y13, Y1, Y9 |
| VPERM2I128 $0x02, Y6, Y10, Y13 |
| VPERM2I128 $0x02, Y8, Y2, Y1 |
| VPERM2I128 $0x13, Y6, Y10, Y6 |
| VPERM2I128 $0x13, Y8, Y2, Y10 |
| JMP openAVX2ShortOpen |
| |
| openAVX2Tail128: |
| // Need to decrypt up to 128 bytes - prepare two blocks |
| VMOVDQA ·chacha20Constants<>+0(SB), Y5 |
| VMOVDQA 32(BP), Y9 |
| VMOVDQA 64(BP), Y13 |
| VMOVDQA 192(BP), Y1 |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y1 |
| VMOVDQA Y1, Y4 |
| XORQ R9, R9 |
| MOVQ BX, CX |
| ANDQ $-16, CX |
| TESTQ CX, CX |
| JE openAVX2Tail128LoopB |
| |
| openAVX2Tail128LoopA: |
| ADDQ (SI)(R9*1), R10 |
| ADCQ 8(SI)(R9*1), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| |
| openAVX2Tail128LoopB: |
| ADDQ $0x10, R9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPALIGNR $0x04, Y9, Y9, Y9 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x0c, Y1, Y1, Y1 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPALIGNR $0x0c, Y9, Y9, Y9 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x04, Y1, Y1, Y1 |
| CMPQ R9, CX |
| JB openAVX2Tail128LoopA |
| CMPQ R9, $0xa0 |
| JNE openAVX2Tail128LoopB |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 |
| VPADDD 32(BP), Y9, Y9 |
| VPADDD 64(BP), Y13, Y13 |
| VPADDD Y4, Y1, Y1 |
| VPERM2I128 $0x02, Y5, Y9, Y0 |
| VPERM2I128 $0x02, Y13, Y1, Y14 |
| VPERM2I128 $0x13, Y5, Y9, Y12 |
| VPERM2I128 $0x13, Y13, Y1, Y4 |
| |
| openAVX2TailLoop: |
| CMPQ BX, $0x20 |
| JB openAVX2Tail |
| SUBQ $0x20, BX |
| |
| // Load for decryption |
| VPXOR (SI), Y0, Y0 |
| VMOVDQU Y0, (DI) |
| LEAQ 32(SI), SI |
| LEAQ 32(DI), DI |
| VMOVDQA Y14, Y0 |
| VMOVDQA Y12, Y14 |
| VMOVDQA Y4, Y12 |
| JMP openAVX2TailLoop |
| |
| openAVX2Tail: |
| CMPQ BX, $0x10 |
| VMOVDQA X0, X1 |
| JB openAVX2TailDone |
| SUBQ $0x10, BX |
| |
| // Load for decryption |
| VPXOR (SI), X0, X12 |
| VMOVDQU X12, (DI) |
| LEAQ 16(SI), SI |
| LEAQ 16(DI), DI |
| VPERM2I128 $0x11, Y0, Y0, Y0 |
| VMOVDQA X0, X1 |
| |
| openAVX2TailDone: |
| VZEROUPPER |
| JMP openSSETail16 |
| |
| openAVX2Tail256: |
| VMOVDQA ·chacha20Constants<>+0(SB), Y0 |
| VMOVDQA Y0, Y5 |
| VMOVDQA 32(BP), Y14 |
| VMOVDQA Y14, Y9 |
| VMOVDQA 64(BP), Y12 |
| VMOVDQA Y12, Y13 |
| VMOVDQA 192(BP), Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 |
| VMOVDQA Y4, Y7 |
| VMOVDQA Y1, Y11 |
| |
| // Compute the number of iterations that will hash data |
| MOVQ BX, 224(BP) |
| MOVQ BX, CX |
| SUBQ $0x80, CX |
| SHRQ $0x04, CX |
| MOVQ $0x0000000a, R9 |
| CMPQ CX, $0x0a |
| CMOVQGT R9, CX |
| MOVQ SI, BX |
| XORQ R9, R9 |
| |
| openAVX2Tail256LoopA: |
| ADDQ (BX), R10 |
| ADCQ 8(BX), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 16(BX), BX |
| |
| openAVX2Tail256LoopB: |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPALIGNR $0x04, Y14, Y14, Y14 |
| VPALIGNR $0x04, Y9, Y9, Y9 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x0c, Y4, Y4, Y4 |
| VPALIGNR $0x0c, Y1, Y1, Y1 |
| INCQ R9 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPALIGNR $0x0c, Y14, Y14, Y14 |
| VPALIGNR $0x0c, Y9, Y9, Y9 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x04, Y4, Y4, Y4 |
| VPALIGNR $0x04, Y1, Y1, Y1 |
| CMPQ R9, CX |
| JB openAVX2Tail256LoopA |
| CMPQ R9, $0x0a |
| JNE openAVX2Tail256LoopB |
| MOVQ BX, R9 |
| SUBQ SI, BX |
| MOVQ BX, CX |
| MOVQ 224(BP), BX |
| |
| openAVX2Tail256Hash: |
| ADDQ $0x10, CX |
| CMPQ CX, BX |
| JGT openAVX2Tail256HashEnd |
| ADDQ (R9), R10 |
| ADCQ 8(R9), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 16(R9), R9 |
| JMP openAVX2Tail256Hash |
| |
| openAVX2Tail256HashEnd: |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 |
| VPADDD 32(BP), Y14, Y14 |
| VPADDD 32(BP), Y9, Y9 |
| VPADDD 64(BP), Y12, Y12 |
| VPADDD 64(BP), Y13, Y13 |
| VPADDD Y7, Y4, Y4 |
| VPADDD Y11, Y1, Y1 |
| VPERM2I128 $0x02, Y0, Y14, Y6 |
| VPERM2I128 $0x02, Y12, Y4, Y10 |
| VPERM2I128 $0x13, Y0, Y14, Y8 |
| VPERM2I128 $0x13, Y12, Y4, Y2 |
| VPERM2I128 $0x02, Y5, Y9, Y0 |
| VPERM2I128 $0x02, Y13, Y1, Y14 |
| VPERM2I128 $0x13, Y5, Y9, Y12 |
| VPERM2I128 $0x13, Y13, Y1, Y4 |
| VPXOR (SI), Y6, Y6 |
| VPXOR 32(SI), Y10, Y10 |
| VPXOR 64(SI), Y8, Y8 |
| VPXOR 96(SI), Y2, Y2 |
| VMOVDQU Y6, (DI) |
| VMOVDQU Y10, 32(DI) |
| VMOVDQU Y8, 64(DI) |
| VMOVDQU Y2, 96(DI) |
| LEAQ 128(SI), SI |
| LEAQ 128(DI), DI |
| SUBQ $0x80, BX |
| JMP openAVX2TailLoop |
| |
| openAVX2Tail384: |
| // Need to decrypt up to 384 bytes - prepare six blocks |
| VMOVDQA ·chacha20Constants<>+0(SB), Y0 |
| VMOVDQA Y0, Y5 |
| VMOVDQA Y0, Y6 |
| VMOVDQA 32(BP), Y14 |
| VMOVDQA Y14, Y9 |
| VMOVDQA Y14, Y10 |
| VMOVDQA 64(BP), Y12 |
| VMOVDQA Y12, Y13 |
| VMOVDQA Y12, Y8 |
| VMOVDQA 192(BP), Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 |
| VMOVDQA Y4, 96(BP) |
| VMOVDQA Y1, 128(BP) |
| VMOVDQA Y2, 160(BP) |
| |
| // Compute the number of iterations that will hash two blocks of data |
| MOVQ BX, 224(BP) |
| MOVQ BX, CX |
| SUBQ $0x00000100, CX |
| SHRQ $0x04, CX |
| ADDQ $0x06, CX |
| MOVQ $0x0000000a, R9 |
| CMPQ CX, $0x0a |
| CMOVQGT R9, CX |
| MOVQ SI, BX |
| XORQ R9, R9 |
| |
| openAVX2Tail384LoopB: |
| ADDQ (BX), R10 |
| ADCQ 8(BX), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 16(BX), BX |
| |
| openAVX2Tail384LoopA: |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x0c, Y10, Y3 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y3, Y10, Y10 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x07, Y10, Y3 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y3, Y10, Y10 |
| VPALIGNR $0x04, Y14, Y14, Y14 |
| VPALIGNR $0x04, Y9, Y9, Y9 |
| VPALIGNR $0x04, Y10, Y10, Y10 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x0c, Y4, Y4, Y4 |
| VPALIGNR $0x0c, Y1, Y1, Y1 |
| VPALIGNR $0x0c, Y2, Y2, Y2 |
| ADDQ (BX), R10 |
| ADCQ 8(BX), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 16(BX), BX |
| INCQ R9 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x0c, Y10, Y3 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y3, Y10, Y10 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x07, Y10, Y3 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y3, Y10, Y10 |
| VPALIGNR $0x0c, Y14, Y14, Y14 |
| VPALIGNR $0x0c, Y9, Y9, Y9 |
| VPALIGNR $0x0c, Y10, Y10, Y10 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x04, Y4, Y4, Y4 |
| VPALIGNR $0x04, Y1, Y1, Y1 |
| VPALIGNR $0x04, Y2, Y2, Y2 |
| CMPQ R9, CX |
| JB openAVX2Tail384LoopB |
| CMPQ R9, $0x0a |
| JNE openAVX2Tail384LoopA |
| MOVQ BX, R9 |
| SUBQ SI, BX |
| MOVQ BX, CX |
| MOVQ 224(BP), BX |
| |
| openAVX2Tail384Hash: |
| ADDQ $0x10, CX |
| CMPQ CX, BX |
| JGT openAVX2Tail384HashEnd |
| ADDQ (R9), R10 |
| ADCQ 8(R9), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 16(R9), R9 |
| JMP openAVX2Tail384Hash |
| |
| openAVX2Tail384HashEnd: |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 |
| VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 |
| VPADDD 32(BP), Y14, Y14 |
| VPADDD 32(BP), Y9, Y9 |
| VPADDD 32(BP), Y10, Y10 |
| VPADDD 64(BP), Y12, Y12 |
| VPADDD 64(BP), Y13, Y13 |
| VPADDD 64(BP), Y8, Y8 |
| VPADDD 96(BP), Y4, Y4 |
| VPADDD 128(BP), Y1, Y1 |
| VPADDD 160(BP), Y2, Y2 |
| VPERM2I128 $0x02, Y0, Y14, Y3 |
| VPERM2I128 $0x02, Y12, Y4, Y7 |
| VPERM2I128 $0x13, Y0, Y14, Y11 |
| VPERM2I128 $0x13, Y12, Y4, Y15 |
| VPXOR (SI), Y3, Y3 |
| VPXOR 32(SI), Y7, Y7 |
| VPXOR 64(SI), Y11, Y11 |
| VPXOR 96(SI), Y15, Y15 |
| VMOVDQU Y3, (DI) |
| VMOVDQU Y7, 32(DI) |
| VMOVDQU Y11, 64(DI) |
| VMOVDQU Y15, 96(DI) |
| VPERM2I128 $0x02, Y5, Y9, Y3 |
| VPERM2I128 $0x02, Y13, Y1, Y7 |
| VPERM2I128 $0x13, Y5, Y9, Y11 |
| VPERM2I128 $0x13, Y13, Y1, Y15 |
| VPXOR 128(SI), Y3, Y3 |
| VPXOR 160(SI), Y7, Y7 |
| VPXOR 192(SI), Y11, Y11 |
| VPXOR 224(SI), Y15, Y15 |
| VMOVDQU Y3, 128(DI) |
| VMOVDQU Y7, 160(DI) |
| VMOVDQU Y11, 192(DI) |
| VMOVDQU Y15, 224(DI) |
| VPERM2I128 $0x02, Y6, Y10, Y0 |
| VPERM2I128 $0x02, Y8, Y2, Y14 |
| VPERM2I128 $0x13, Y6, Y10, Y12 |
| VPERM2I128 $0x13, Y8, Y2, Y4 |
| LEAQ 256(SI), SI |
| LEAQ 256(DI), DI |
| SUBQ $0x00000100, BX |
| JMP openAVX2TailLoop |
| |
| openAVX2Tail512: |
| VMOVDQU ·chacha20Constants<>+0(SB), Y0 |
| VMOVDQA Y0, Y5 |
| VMOVDQA Y0, Y6 |
| VMOVDQA Y0, Y7 |
| VMOVDQA 32(BP), Y14 |
| VMOVDQA Y14, Y9 |
| VMOVDQA Y14, Y10 |
| VMOVDQA Y14, Y11 |
| VMOVDQA 64(BP), Y12 |
| VMOVDQA Y12, Y13 |
| VMOVDQA Y12, Y8 |
| VMOVDQA Y12, Y15 |
| VMOVDQA 192(BP), Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 |
| VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 |
| VMOVDQA Y4, 96(BP) |
| VMOVDQA Y1, 128(BP) |
| VMOVDQA Y2, 160(BP) |
| VMOVDQA Y3, 192(BP) |
| XORQ CX, CX |
| MOVQ SI, R9 |
| |
| openAVX2Tail512LoopB: |
| ADDQ (R9), R10 |
| ADCQ 8(R9), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 16(R9), R9 |
| |
| openAVX2Tail512LoopA: |
| VPADDD Y14, Y0, Y0 |
| VPADDD Y9, Y5, Y5 |
| VPADDD Y10, Y6, Y6 |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y0, Y4, Y4 |
| VPXOR Y5, Y1, Y1 |
| VPXOR Y6, Y2, Y2 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 |
| VPADDD Y4, Y12, Y12 |
| VPADDD Y1, Y13, Y13 |
| VPADDD Y2, Y8, Y8 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y12, Y14, Y14 |
| VPXOR Y13, Y9, Y9 |
| VPXOR Y8, Y10, Y10 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA Y15, 224(BP) |
| VPSLLD $0x0c, Y14, Y15 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPSLLD $0x0c, Y9, Y15 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPSLLD $0x0c, Y10, Y15 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPSLLD $0x0c, Y11, Y15 |
| VPSRLD $0x14, Y11, Y11 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA 224(BP), Y15 |
| ADDQ (R9), R10 |
| ADCQ 8(R9), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| VPADDD Y14, Y0, Y0 |
| VPADDD Y9, Y5, Y5 |
| VPADDD Y10, Y6, Y6 |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y0, Y4, Y4 |
| VPXOR Y5, Y1, Y1 |
| VPXOR Y6, Y2, Y2 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 |
| VPADDD Y4, Y12, Y12 |
| VPADDD Y1, Y13, Y13 |
| VPADDD Y2, Y8, Y8 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y12, Y14, Y14 |
| VPXOR Y13, Y9, Y9 |
| VPXOR Y8, Y10, Y10 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA Y15, 224(BP) |
| VPSLLD $0x07, Y14, Y15 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPSLLD $0x07, Y9, Y15 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPSLLD $0x07, Y10, Y15 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPSLLD $0x07, Y11, Y15 |
| VPSRLD $0x19, Y11, Y11 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA 224(BP), Y15 |
| VPALIGNR $0x04, Y14, Y14, Y14 |
| VPALIGNR $0x04, Y9, Y9, Y9 |
| VPALIGNR $0x04, Y10, Y10, Y10 |
| VPALIGNR $0x04, Y11, Y11, Y11 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x08, Y15, Y15, Y15 |
| VPALIGNR $0x0c, Y4, Y4, Y4 |
| VPALIGNR $0x0c, Y1, Y1, Y1 |
| VPALIGNR $0x0c, Y2, Y2, Y2 |
| VPALIGNR $0x0c, Y3, Y3, Y3 |
| VPADDD Y14, Y0, Y0 |
| VPADDD Y9, Y5, Y5 |
| VPADDD Y10, Y6, Y6 |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y0, Y4, Y4 |
| VPXOR Y5, Y1, Y1 |
| VPXOR Y6, Y2, Y2 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 |
| VPADDD Y4, Y12, Y12 |
| VPADDD Y1, Y13, Y13 |
| VPADDD Y2, Y8, Y8 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y12, Y14, Y14 |
| VPXOR Y13, Y9, Y9 |
| VPXOR Y8, Y10, Y10 |
| VPXOR Y15, Y11, Y11 |
| ADDQ 16(R9), R10 |
| ADCQ 24(R9), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 32(R9), R9 |
| VMOVDQA Y15, 224(BP) |
| VPSLLD $0x0c, Y14, Y15 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPSLLD $0x0c, Y9, Y15 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPSLLD $0x0c, Y10, Y15 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPSLLD $0x0c, Y11, Y15 |
| VPSRLD $0x14, Y11, Y11 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA 224(BP), Y15 |
| VPADDD Y14, Y0, Y0 |
| VPADDD Y9, Y5, Y5 |
| VPADDD Y10, Y6, Y6 |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y0, Y4, Y4 |
| VPXOR Y5, Y1, Y1 |
| VPXOR Y6, Y2, Y2 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 |
| VPADDD Y4, Y12, Y12 |
| VPADDD Y1, Y13, Y13 |
| VPADDD Y2, Y8, Y8 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y12, Y14, Y14 |
| VPXOR Y13, Y9, Y9 |
| VPXOR Y8, Y10, Y10 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA Y15, 224(BP) |
| VPSLLD $0x07, Y14, Y15 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPSLLD $0x07, Y9, Y15 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPSLLD $0x07, Y10, Y15 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPSLLD $0x07, Y11, Y15 |
| VPSRLD $0x19, Y11, Y11 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA 224(BP), Y15 |
| VPALIGNR $0x0c, Y14, Y14, Y14 |
| VPALIGNR $0x0c, Y9, Y9, Y9 |
| VPALIGNR $0x0c, Y10, Y10, Y10 |
| VPALIGNR $0x0c, Y11, Y11, Y11 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x08, Y15, Y15, Y15 |
| VPALIGNR $0x04, Y4, Y4, Y4 |
| VPALIGNR $0x04, Y1, Y1, Y1 |
| VPALIGNR $0x04, Y2, Y2, Y2 |
| VPALIGNR $0x04, Y3, Y3, Y3 |
| INCQ CX |
| CMPQ CX, $0x04 |
| JLT openAVX2Tail512LoopB |
| CMPQ CX, $0x0a |
| JNE openAVX2Tail512LoopA |
| MOVQ BX, CX |
| SUBQ $0x00000180, CX |
| ANDQ $-16, CX |
| |
| openAVX2Tail512HashLoop: |
| TESTQ CX, CX |
| JE openAVX2Tail512HashEnd |
| ADDQ (R9), R10 |
| ADCQ 8(R9), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 16(R9), R9 |
| SUBQ $0x10, CX |
| JMP openAVX2Tail512HashLoop |
| |
| openAVX2Tail512HashEnd: |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 |
| VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 |
| VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 |
| VPADDD 32(BP), Y14, Y14 |
| VPADDD 32(BP), Y9, Y9 |
| VPADDD 32(BP), Y10, Y10 |
| VPADDD 32(BP), Y11, Y11 |
| VPADDD 64(BP), Y12, Y12 |
| VPADDD 64(BP), Y13, Y13 |
| VPADDD 64(BP), Y8, Y8 |
| VPADDD 64(BP), Y15, Y15 |
| VPADDD 96(BP), Y4, Y4 |
| VPADDD 128(BP), Y1, Y1 |
| VPADDD 160(BP), Y2, Y2 |
| VPADDD 192(BP), Y3, Y3 |
| VMOVDQA Y15, 224(BP) |
| VPERM2I128 $0x02, Y0, Y14, Y15 |
| VPERM2I128 $0x13, Y0, Y14, Y14 |
| VPERM2I128 $0x02, Y12, Y4, Y0 |
| VPERM2I128 $0x13, Y12, Y4, Y12 |
| VPXOR (SI), Y15, Y15 |
| VPXOR 32(SI), Y0, Y0 |
| VPXOR 64(SI), Y14, Y14 |
| VPXOR 96(SI), Y12, Y12 |
| VMOVDQU Y15, (DI) |
| VMOVDQU Y0, 32(DI) |
| VMOVDQU Y14, 64(DI) |
| VMOVDQU Y12, 96(DI) |
| VPERM2I128 $0x02, Y5, Y9, Y0 |
| VPERM2I128 $0x02, Y13, Y1, Y14 |
| VPERM2I128 $0x13, Y5, Y9, Y12 |
| VPERM2I128 $0x13, Y13, Y1, Y4 |
| VPXOR 128(SI), Y0, Y0 |
| VPXOR 160(SI), Y14, Y14 |
| VPXOR 192(SI), Y12, Y12 |
| VPXOR 224(SI), Y4, Y4 |
| VMOVDQU Y0, 128(DI) |
| VMOVDQU Y14, 160(DI) |
| VMOVDQU Y12, 192(DI) |
| VMOVDQU Y4, 224(DI) |
| VPERM2I128 $0x02, Y6, Y10, Y0 |
| VPERM2I128 $0x02, Y8, Y2, Y14 |
| VPERM2I128 $0x13, Y6, Y10, Y12 |
| VPERM2I128 $0x13, Y8, Y2, Y4 |
| VPXOR 256(SI), Y0, Y0 |
| VPXOR 288(SI), Y14, Y14 |
| VPXOR 320(SI), Y12, Y12 |
| VPXOR 352(SI), Y4, Y4 |
| VMOVDQU Y0, 256(DI) |
| VMOVDQU Y14, 288(DI) |
| VMOVDQU Y12, 320(DI) |
| VMOVDQU Y4, 352(DI) |
| VPERM2I128 $0x02, Y7, Y11, Y0 |
| VPERM2I128 $0x02, 224(BP), Y3, Y14 |
| VPERM2I128 $0x13, Y7, Y11, Y12 |
| VPERM2I128 $0x13, 224(BP), Y3, Y4 |
| LEAQ 384(SI), SI |
| LEAQ 384(DI), DI |
| SUBQ $0x00000180, BX |
| JMP openAVX2TailLoop |
| |
| DATA ·chacha20Constants<>+0(SB)/4, $0x61707865 |
| DATA ·chacha20Constants<>+4(SB)/4, $0x3320646e |
| DATA ·chacha20Constants<>+8(SB)/4, $0x79622d32 |
| DATA ·chacha20Constants<>+12(SB)/4, $0x6b206574 |
| DATA ·chacha20Constants<>+16(SB)/4, $0x61707865 |
| DATA ·chacha20Constants<>+20(SB)/4, $0x3320646e |
| DATA ·chacha20Constants<>+24(SB)/4, $0x79622d32 |
| DATA ·chacha20Constants<>+28(SB)/4, $0x6b206574 |
| GLOBL ·chacha20Constants<>(SB), RODATA|NOPTR, $32 |
| |
| DATA ·avx2InitMask<>+0(SB)/8, $0x0000000000000000 |
| DATA ·avx2InitMask<>+8(SB)/8, $0x0000000000000000 |
| DATA ·avx2InitMask<>+16(SB)/8, $0x0000000000000001 |
| DATA ·avx2InitMask<>+24(SB)/8, $0x0000000000000000 |
| GLOBL ·avx2InitMask<>(SB), RODATA|NOPTR, $32 |
| |
| DATA ·rol16<>+0(SB)/8, $0x0504070601000302 |
| DATA ·rol16<>+8(SB)/8, $0x0d0c0f0e09080b0a |
| DATA ·rol16<>+16(SB)/8, $0x0504070601000302 |
| DATA ·rol16<>+24(SB)/8, $0x0d0c0f0e09080b0a |
| GLOBL ·rol16<>(SB), RODATA|NOPTR, $32 |
| |
| DATA ·rol8<>+0(SB)/8, $0x0605040702010003 |
| DATA ·rol8<>+8(SB)/8, $0x0e0d0c0f0a09080b |
| DATA ·rol8<>+16(SB)/8, $0x0605040702010003 |
| DATA ·rol8<>+24(SB)/8, $0x0e0d0c0f0a09080b |
| GLOBL ·rol8<>(SB), RODATA|NOPTR, $32 |
| |
| DATA ·polyClampMask<>+0(SB)/8, $0x0ffffffc0fffffff |
| DATA ·polyClampMask<>+8(SB)/8, $0x0ffffffc0ffffffc |
| DATA ·polyClampMask<>+16(SB)/8, $0xffffffffffffffff |
| DATA ·polyClampMask<>+24(SB)/8, $0xffffffffffffffff |
| GLOBL ·polyClampMask<>(SB), RODATA|NOPTR, $32 |
| |
| DATA ·avx2IncMask<>+0(SB)/8, $0x0000000000000002 |
| DATA ·avx2IncMask<>+8(SB)/8, $0x0000000000000000 |
| DATA ·avx2IncMask<>+16(SB)/8, $0x0000000000000002 |
| DATA ·avx2IncMask<>+24(SB)/8, $0x0000000000000000 |
| GLOBL ·avx2IncMask<>(SB), RODATA|NOPTR, $32 |
| |
| DATA ·andMask<>+0(SB)/8, $0x00000000000000ff |
| DATA ·andMask<>+8(SB)/8, $0x0000000000000000 |
| DATA ·andMask<>+16(SB)/8, $0x000000000000ffff |
| DATA ·andMask<>+24(SB)/8, $0x0000000000000000 |
| DATA ·andMask<>+32(SB)/8, $0x0000000000ffffff |
| DATA ·andMask<>+40(SB)/8, $0x0000000000000000 |
| DATA ·andMask<>+48(SB)/8, $0x00000000ffffffff |
| DATA ·andMask<>+56(SB)/8, $0x0000000000000000 |
| DATA ·andMask<>+64(SB)/8, $0x000000ffffffffff |
| DATA ·andMask<>+72(SB)/8, $0x0000000000000000 |
| DATA ·andMask<>+80(SB)/8, $0x0000ffffffffffff |
| DATA ·andMask<>+88(SB)/8, $0x0000000000000000 |
| DATA ·andMask<>+96(SB)/8, $0x00ffffffffffffff |
| DATA ·andMask<>+104(SB)/8, $0x0000000000000000 |
| DATA ·andMask<>+112(SB)/8, $0xffffffffffffffff |
| DATA ·andMask<>+120(SB)/8, $0x0000000000000000 |
| DATA ·andMask<>+128(SB)/8, $0xffffffffffffffff |
| DATA ·andMask<>+136(SB)/8, $0x00000000000000ff |
| DATA ·andMask<>+144(SB)/8, $0xffffffffffffffff |
| DATA ·andMask<>+152(SB)/8, $0x000000000000ffff |
| DATA ·andMask<>+160(SB)/8, $0xffffffffffffffff |
| DATA ·andMask<>+168(SB)/8, $0x0000000000ffffff |
| DATA ·andMask<>+176(SB)/8, $0xffffffffffffffff |
| DATA ·andMask<>+184(SB)/8, $0x00000000ffffffff |
| DATA ·andMask<>+192(SB)/8, $0xffffffffffffffff |
| DATA ·andMask<>+200(SB)/8, $0x000000ffffffffff |
| DATA ·andMask<>+208(SB)/8, $0xffffffffffffffff |
| DATA ·andMask<>+216(SB)/8, $0x0000ffffffffffff |
| DATA ·andMask<>+224(SB)/8, $0xffffffffffffffff |
| DATA ·andMask<>+232(SB)/8, $0x00ffffffffffffff |
| GLOBL ·andMask<>(SB), RODATA|NOPTR, $240 |
| |
| // func chacha20Poly1305Seal(dst []byte, key []uint32, src []byte, ad []byte) |
| // Requires: AVX, AVX2, BMI2, CMOV, SSE2 |
| TEXT ·chacha20Poly1305Seal(SB), $288-96 |
| MOVQ SP, BP |
| ADDQ $0x20, BP |
| ANDQ $-32, BP |
| MOVQ dst_base+0(FP), DI |
| MOVQ key_base+24(FP), R8 |
| MOVQ src_base+48(FP), SI |
| MOVQ src_len+56(FP), BX |
| MOVQ ad_base+72(FP), CX |
| VZEROUPPER |
| VMOVDQU ·chacha20Constants<>+0(SB), Y0 |
| VBROADCASTI128 16(R8), Y14 |
| VBROADCASTI128 32(R8), Y12 |
| VBROADCASTI128 48(R8), Y4 |
| VPADDD ·avx2InitMask<>+0(SB), Y4, Y4 |
| |
| // Special optimizations, for very short buffers |
| CMPQ BX, $0x000000c0 |
| JBE seal192AVX2 |
| CMPQ BX, $0x00000140 |
| JBE seal320AVX2 |
| |
| // For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream |
| VMOVDQA Y0, Y5 |
| VMOVDQA Y0, Y6 |
| VMOVDQA Y0, Y7 |
| VMOVDQA Y14, Y9 |
| VMOVDQA Y14, Y10 |
| VMOVDQA Y14, Y11 |
| VMOVDQA Y14, 32(BP) |
| VMOVDQA Y12, Y13 |
| VMOVDQA Y12, Y8 |
| VMOVDQA Y12, Y15 |
| VMOVDQA Y12, 64(BP) |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 |
| VMOVDQA Y4, 96(BP) |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 |
| VMOVDQA Y1, 128(BP) |
| VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 |
| VMOVDQA Y2, 160(BP) |
| VMOVDQA Y3, 192(BP) |
| MOVQ $0x0000000a, R9 |
| |
| sealAVX2IntroLoop: |
| VMOVDQA Y15, 224(BP) |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y15 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y15 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y15 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y15 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x0c, Y10, Y15 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x07, Y10, Y15 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VMOVDQA 224(BP), Y15 |
| VMOVDQA Y13, 224(BP) |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y15, Y11, Y11 |
| VPSLLD $0x0c, Y11, Y13 |
| VPSRLD $0x14, Y11, Y11 |
| VPXOR Y13, Y11, Y11 |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y15, Y11, Y11 |
| VPSLLD $0x07, Y11, Y13 |
| VPSRLD $0x19, Y11, Y11 |
| VPXOR Y13, Y11, Y11 |
| VMOVDQA 224(BP), Y13 |
| VPALIGNR $0x04, Y14, Y14, Y14 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x0c, Y4, Y4, Y4 |
| VPALIGNR $0x04, Y9, Y9, Y9 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x0c, Y1, Y1, Y1 |
| VPALIGNR $0x04, Y10, Y10, Y10 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x0c, Y2, Y2, Y2 |
| VPALIGNR $0x04, Y11, Y11, Y11 |
| VPALIGNR $0x08, Y15, Y15, Y15 |
| VPALIGNR $0x0c, Y3, Y3, Y3 |
| VMOVDQA Y15, 224(BP) |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y15 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y15 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y15 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y15 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x0c, Y10, Y15 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x07, Y10, Y15 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VMOVDQA 224(BP), Y15 |
| VMOVDQA Y13, 224(BP) |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y15, Y11, Y11 |
| VPSLLD $0x0c, Y11, Y13 |
| VPSRLD $0x14, Y11, Y11 |
| VPXOR Y13, Y11, Y11 |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y15, Y11, Y11 |
| VPSLLD $0x07, Y11, Y13 |
| VPSRLD $0x19, Y11, Y11 |
| VPXOR Y13, Y11, Y11 |
| VMOVDQA 224(BP), Y13 |
| VPALIGNR $0x0c, Y14, Y14, Y14 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x04, Y4, Y4, Y4 |
| VPALIGNR $0x0c, Y9, Y9, Y9 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x04, Y1, Y1, Y1 |
| VPALIGNR $0x0c, Y10, Y10, Y10 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x04, Y2, Y2, Y2 |
| VPALIGNR $0x0c, Y11, Y11, Y11 |
| VPALIGNR $0x08, Y15, Y15, Y15 |
| VPALIGNR $0x04, Y3, Y3, Y3 |
| DECQ R9 |
| JNE sealAVX2IntroLoop |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 |
| VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 |
| VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 |
| VPADDD 32(BP), Y14, Y14 |
| VPADDD 32(BP), Y9, Y9 |
| VPADDD 32(BP), Y10, Y10 |
| VPADDD 32(BP), Y11, Y11 |
| VPADDD 64(BP), Y12, Y12 |
| VPADDD 64(BP), Y13, Y13 |
| VPADDD 64(BP), Y8, Y8 |
| VPADDD 64(BP), Y15, Y15 |
| VPADDD 96(BP), Y4, Y4 |
| VPADDD 128(BP), Y1, Y1 |
| VPADDD 160(BP), Y2, Y2 |
| VPADDD 192(BP), Y3, Y3 |
| VPERM2I128 $0x13, Y12, Y4, Y12 |
| VPERM2I128 $0x02, Y0, Y14, Y4 |
| VPERM2I128 $0x13, Y0, Y14, Y0 |
| |
| // Clamp and store poly key |
| VPAND ·polyClampMask<>+0(SB), Y4, Y4 |
| VMOVDQA Y4, (BP) |
| |
| // Hash AD |
| MOVQ ad_len+80(FP), R9 |
| CALL polyHashADInternal<>(SB) |
| |
| // Can store at least 320 bytes |
| VPXOR (SI), Y0, Y0 |
| VPXOR 32(SI), Y12, Y12 |
| VMOVDQU Y0, (DI) |
| VMOVDQU Y12, 32(DI) |
| VPERM2I128 $0x02, Y5, Y9, Y0 |
| VPERM2I128 $0x02, Y13, Y1, Y14 |
| VPERM2I128 $0x13, Y5, Y9, Y12 |
| VPERM2I128 $0x13, Y13, Y1, Y4 |
| VPXOR 64(SI), Y0, Y0 |
| VPXOR 96(SI), Y14, Y14 |
| VPXOR 128(SI), Y12, Y12 |
| VPXOR 160(SI), Y4, Y4 |
| VMOVDQU Y0, 64(DI) |
| VMOVDQU Y14, 96(DI) |
| VMOVDQU Y12, 128(DI) |
| VMOVDQU Y4, 160(DI) |
| VPERM2I128 $0x02, Y6, Y10, Y0 |
| VPERM2I128 $0x02, Y8, Y2, Y14 |
| VPERM2I128 $0x13, Y6, Y10, Y12 |
| VPERM2I128 $0x13, Y8, Y2, Y4 |
| VPXOR 192(SI), Y0, Y0 |
| VPXOR 224(SI), Y14, Y14 |
| VPXOR 256(SI), Y12, Y12 |
| VPXOR 288(SI), Y4, Y4 |
| VMOVDQU Y0, 192(DI) |
| VMOVDQU Y14, 224(DI) |
| VMOVDQU Y12, 256(DI) |
| VMOVDQU Y4, 288(DI) |
| MOVQ $0x00000140, CX |
| SUBQ $0x00000140, BX |
| LEAQ 320(SI), SI |
| VPERM2I128 $0x02, Y7, Y11, Y0 |
| VPERM2I128 $0x02, Y15, Y3, Y14 |
| VPERM2I128 $0x13, Y7, Y11, Y12 |
| VPERM2I128 $0x13, Y15, Y3, Y4 |
| CMPQ BX, $0x80 |
| JBE sealAVX2SealHash |
| VPXOR (SI), Y0, Y0 |
| VPXOR 32(SI), Y14, Y14 |
| VPXOR 64(SI), Y12, Y12 |
| VPXOR 96(SI), Y4, Y4 |
| VMOVDQU Y0, 320(DI) |
| VMOVDQU Y14, 352(DI) |
| VMOVDQU Y12, 384(DI) |
| VMOVDQU Y4, 416(DI) |
| SUBQ $0x80, BX |
| LEAQ 128(SI), SI |
| MOVQ $0x00000008, CX |
| MOVQ $0x00000002, R9 |
| CMPQ BX, $0x80 |
| JBE sealAVX2Tail128 |
| CMPQ BX, $0x00000100 |
| JBE sealAVX2Tail256 |
| CMPQ BX, $0x00000180 |
| JBE sealAVX2Tail384 |
| CMPQ BX, $0x00000200 |
| JBE sealAVX2Tail512 |
| |
| // We have 448 bytes to hash, but main loop hashes 512 bytes at a time - perform some rounds, before the main loop |
| VMOVDQA ·chacha20Constants<>+0(SB), Y0 |
| VMOVDQA Y0, Y5 |
| VMOVDQA Y0, Y6 |
| VMOVDQA Y0, Y7 |
| VMOVDQA 32(BP), Y14 |
| VMOVDQA Y14, Y9 |
| VMOVDQA Y14, Y10 |
| VMOVDQA Y14, Y11 |
| VMOVDQA 64(BP), Y12 |
| VMOVDQA Y12, Y13 |
| VMOVDQA Y12, Y8 |
| VMOVDQA Y12, Y15 |
| VMOVDQA 192(BP), Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 |
| VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 |
| VMOVDQA Y4, 96(BP) |
| VMOVDQA Y1, 128(BP) |
| VMOVDQA Y2, 160(BP) |
| VMOVDQA Y3, 192(BP) |
| VMOVDQA Y15, 224(BP) |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y15 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y15 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y15 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y15 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x0c, Y10, Y15 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x07, Y10, Y15 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VMOVDQA 224(BP), Y15 |
| VMOVDQA Y13, 224(BP) |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y15, Y11, Y11 |
| VPSLLD $0x0c, Y11, Y13 |
| VPSRLD $0x14, Y11, Y11 |
| VPXOR Y13, Y11, Y11 |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y15, Y11, Y11 |
| VPSLLD $0x07, Y11, Y13 |
| VPSRLD $0x19, Y11, Y11 |
| VPXOR Y13, Y11, Y11 |
| VMOVDQA 224(BP), Y13 |
| VPALIGNR $0x04, Y14, Y14, Y14 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x0c, Y4, Y4, Y4 |
| VPALIGNR $0x04, Y9, Y9, Y9 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x0c, Y1, Y1, Y1 |
| VPALIGNR $0x04, Y10, Y10, Y10 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x0c, Y2, Y2, Y2 |
| VPALIGNR $0x04, Y11, Y11, Y11 |
| VPALIGNR $0x08, Y15, Y15, Y15 |
| VPALIGNR $0x0c, Y3, Y3, Y3 |
| VMOVDQA Y15, 224(BP) |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y15 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y15 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y15 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y15 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x0c, Y10, Y15 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x07, Y10, Y15 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VMOVDQA 224(BP), Y15 |
| VMOVDQA Y13, 224(BP) |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y15, Y11, Y11 |
| VPSLLD $0x0c, Y11, Y13 |
| VPSRLD $0x14, Y11, Y11 |
| VPXOR Y13, Y11, Y11 |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y15, Y11, Y11 |
| VPSLLD $0x07, Y11, Y13 |
| VPSRLD $0x19, Y11, Y11 |
| VPXOR Y13, Y11, Y11 |
| VMOVDQA 224(BP), Y13 |
| VPALIGNR $0x0c, Y14, Y14, Y14 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x04, Y4, Y4, Y4 |
| VPALIGNR $0x0c, Y9, Y9, Y9 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x04, Y1, Y1, Y1 |
| VPALIGNR $0x0c, Y10, Y10, Y10 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x04, Y2, Y2, Y2 |
| VPALIGNR $0x0c, Y11, Y11, Y11 |
| VPALIGNR $0x08, Y15, Y15, Y15 |
| VPALIGNR $0x04, Y3, Y3, Y3 |
| VPADDD Y14, Y0, Y0 |
| VPADDD Y9, Y5, Y5 |
| VPADDD Y10, Y6, Y6 |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y0, Y4, Y4 |
| VPXOR Y5, Y1, Y1 |
| VPXOR Y6, Y2, Y2 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 |
| VPADDD Y4, Y12, Y12 |
| VPADDD Y1, Y13, Y13 |
| VPADDD Y2, Y8, Y8 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y12, Y14, Y14 |
| VPXOR Y13, Y9, Y9 |
| VPXOR Y8, Y10, Y10 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA Y15, 224(BP) |
| VPSLLD $0x0c, Y14, Y15 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPSLLD $0x0c, Y9, Y15 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPSLLD $0x0c, Y10, Y15 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPSLLD $0x0c, Y11, Y15 |
| VPSRLD $0x14, Y11, Y11 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA 224(BP), Y15 |
| SUBQ $0x10, DI |
| MOVQ $0x00000009, CX |
| JMP sealAVX2InternalLoopStart |
| |
| sealAVX2MainLoop: |
| VMOVDQU ·chacha20Constants<>+0(SB), Y0 |
| VMOVDQA Y0, Y5 |
| VMOVDQA Y0, Y6 |
| VMOVDQA Y0, Y7 |
| VMOVDQA 32(BP), Y14 |
| VMOVDQA Y14, Y9 |
| VMOVDQA Y14, Y10 |
| VMOVDQA Y14, Y11 |
| VMOVDQA 64(BP), Y12 |
| VMOVDQA Y12, Y13 |
| VMOVDQA Y12, Y8 |
| VMOVDQA Y12, Y15 |
| VMOVDQA 192(BP), Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 |
| VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 |
| VMOVDQA Y4, 96(BP) |
| VMOVDQA Y1, 128(BP) |
| VMOVDQA Y2, 160(BP) |
| VMOVDQA Y3, 192(BP) |
| MOVQ $0x0000000a, CX |
| |
| sealAVX2InternalLoop: |
| ADDQ (DI), R10 |
| ADCQ 8(DI), R11 |
| ADCQ $0x01, R12 |
| VPADDD Y14, Y0, Y0 |
| VPADDD Y9, Y5, Y5 |
| VPADDD Y10, Y6, Y6 |
| VPADDD Y11, Y7, Y7 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| VPXOR Y0, Y4, Y4 |
| VPXOR Y5, Y1, Y1 |
| VPXOR Y6, Y2, Y2 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| VPADDD Y4, Y12, Y12 |
| VPADDD Y1, Y13, Y13 |
| VPADDD Y2, Y8, Y8 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y12, Y14, Y14 |
| VPXOR Y13, Y9, Y9 |
| VPXOR Y8, Y10, Y10 |
| VPXOR Y15, Y11, Y11 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| VMOVDQA Y15, 224(BP) |
| VPSLLD $0x0c, Y14, Y15 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPSLLD $0x0c, Y9, Y15 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPSLLD $0x0c, Y10, Y15 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPSLLD $0x0c, Y11, Y15 |
| VPSRLD $0x14, Y11, Y11 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA 224(BP), Y15 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| |
| sealAVX2InternalLoopStart: |
| VPADDD Y14, Y0, Y0 |
| VPADDD Y9, Y5, Y5 |
| VPADDD Y10, Y6, Y6 |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y0, Y4, Y4 |
| VPXOR Y5, Y1, Y1 |
| VPXOR Y6, Y2, Y2 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 |
| ADDQ 16(DI), R10 |
| ADCQ 24(DI), R11 |
| ADCQ $0x01, R12 |
| VPADDD Y4, Y12, Y12 |
| VPADDD Y1, Y13, Y13 |
| VPADDD Y2, Y8, Y8 |
| VPADDD Y3, Y15, Y15 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| VPXOR Y12, Y14, Y14 |
| VPXOR Y13, Y9, Y9 |
| VPXOR Y8, Y10, Y10 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA Y15, 224(BP) |
| VPSLLD $0x07, Y14, Y15 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPSLLD $0x07, Y9, Y15 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPSLLD $0x07, Y10, Y15 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPSLLD $0x07, Y11, Y15 |
| VPSRLD $0x19, Y11, Y11 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA 224(BP), Y15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| VPALIGNR $0x04, Y14, Y14, Y14 |
| VPALIGNR $0x04, Y9, Y9, Y9 |
| VPALIGNR $0x04, Y10, Y10, Y10 |
| VPALIGNR $0x04, Y11, Y11, Y11 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x08, Y15, Y15, Y15 |
| VPALIGNR $0x0c, Y4, Y4, Y4 |
| VPALIGNR $0x0c, Y1, Y1, Y1 |
| VPALIGNR $0x0c, Y2, Y2, Y2 |
| VPALIGNR $0x0c, Y3, Y3, Y3 |
| VPADDD Y14, Y0, Y0 |
| VPADDD Y9, Y5, Y5 |
| VPADDD Y10, Y6, Y6 |
| VPADDD Y11, Y7, Y7 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| VPXOR Y0, Y4, Y4 |
| VPXOR Y5, Y1, Y1 |
| VPXOR Y6, Y2, Y2 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| VPADDD Y4, Y12, Y12 |
| VPADDD Y1, Y13, Y13 |
| VPADDD Y2, Y8, Y8 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y12, Y14, Y14 |
| VPXOR Y13, Y9, Y9 |
| VPXOR Y8, Y10, Y10 |
| VPXOR Y15, Y11, Y11 |
| ADDQ 32(DI), R10 |
| ADCQ 40(DI), R11 |
| ADCQ $0x01, R12 |
| LEAQ 48(DI), DI |
| VMOVDQA Y15, 224(BP) |
| VPSLLD $0x0c, Y14, Y15 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPSLLD $0x0c, Y9, Y15 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPSLLD $0x0c, Y10, Y15 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPSLLD $0x0c, Y11, Y15 |
| VPSRLD $0x14, Y11, Y11 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA 224(BP), Y15 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| VPADDD Y14, Y0, Y0 |
| VPADDD Y9, Y5, Y5 |
| VPADDD Y10, Y6, Y6 |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y0, Y4, Y4 |
| VPXOR Y5, Y1, Y1 |
| VPXOR Y6, Y2, Y2 |
| VPXOR Y7, Y3, Y3 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 |
| VPADDD Y4, Y12, Y12 |
| VPADDD Y1, Y13, Y13 |
| VPADDD Y2, Y8, Y8 |
| VPADDD Y3, Y15, Y15 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| VPXOR Y12, Y14, Y14 |
| VPXOR Y13, Y9, Y9 |
| VPXOR Y8, Y10, Y10 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA Y15, 224(BP) |
| VPSLLD $0x07, Y14, Y15 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPSLLD $0x07, Y9, Y15 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPSLLD $0x07, Y10, Y15 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPSLLD $0x07, Y11, Y15 |
| VPSRLD $0x19, Y11, Y11 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA 224(BP), Y15 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| VPALIGNR $0x0c, Y14, Y14, Y14 |
| VPALIGNR $0x0c, Y9, Y9, Y9 |
| VPALIGNR $0x0c, Y10, Y10, Y10 |
| VPALIGNR $0x0c, Y11, Y11, Y11 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x08, Y15, Y15, Y15 |
| VPALIGNR $0x04, Y4, Y4, Y4 |
| VPALIGNR $0x04, Y1, Y1, Y1 |
| VPALIGNR $0x04, Y2, Y2, Y2 |
| VPALIGNR $0x04, Y3, Y3, Y3 |
| DECQ CX |
| JNE sealAVX2InternalLoop |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 |
| VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 |
| VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 |
| VPADDD 32(BP), Y14, Y14 |
| VPADDD 32(BP), Y9, Y9 |
| VPADDD 32(BP), Y10, Y10 |
| VPADDD 32(BP), Y11, Y11 |
| VPADDD 64(BP), Y12, Y12 |
| VPADDD 64(BP), Y13, Y13 |
| VPADDD 64(BP), Y8, Y8 |
| VPADDD 64(BP), Y15, Y15 |
| VPADDD 96(BP), Y4, Y4 |
| VPADDD 128(BP), Y1, Y1 |
| VPADDD 160(BP), Y2, Y2 |
| VPADDD 192(BP), Y3, Y3 |
| VMOVDQA Y15, 224(BP) |
| |
| // We only hashed 480 of the 512 bytes available - hash the remaining 32 here |
| ADDQ (DI), R10 |
| ADCQ 8(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 32(DI), DI |
| VPERM2I128 $0x02, Y0, Y14, Y15 |
| VPERM2I128 $0x13, Y0, Y14, Y14 |
| VPERM2I128 $0x02, Y12, Y4, Y0 |
| VPERM2I128 $0x13, Y12, Y4, Y12 |
| VPXOR (SI), Y15, Y15 |
| VPXOR 32(SI), Y0, Y0 |
| VPXOR 64(SI), Y14, Y14 |
| VPXOR 96(SI), Y12, Y12 |
| VMOVDQU Y15, (DI) |
| VMOVDQU Y0, 32(DI) |
| VMOVDQU Y14, 64(DI) |
| VMOVDQU Y12, 96(DI) |
| VPERM2I128 $0x02, Y5, Y9, Y0 |
| VPERM2I128 $0x02, Y13, Y1, Y14 |
| VPERM2I128 $0x13, Y5, Y9, Y12 |
| VPERM2I128 $0x13, Y13, Y1, Y4 |
| VPXOR 128(SI), Y0, Y0 |
| VPXOR 160(SI), Y14, Y14 |
| VPXOR 192(SI), Y12, Y12 |
| VPXOR 224(SI), Y4, Y4 |
| VMOVDQU Y0, 128(DI) |
| VMOVDQU Y14, 160(DI) |
| VMOVDQU Y12, 192(DI) |
| VMOVDQU Y4, 224(DI) |
| |
| // and here |
| ADDQ -16(DI), R10 |
| ADCQ -8(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| VPERM2I128 $0x02, Y6, Y10, Y0 |
| VPERM2I128 $0x02, Y8, Y2, Y14 |
| VPERM2I128 $0x13, Y6, Y10, Y12 |
| VPERM2I128 $0x13, Y8, Y2, Y4 |
| VPXOR 256(SI), Y0, Y0 |
| VPXOR 288(SI), Y14, Y14 |
| VPXOR 320(SI), Y12, Y12 |
| VPXOR 352(SI), Y4, Y4 |
| VMOVDQU Y0, 256(DI) |
| VMOVDQU Y14, 288(DI) |
| VMOVDQU Y12, 320(DI) |
| VMOVDQU Y4, 352(DI) |
| VPERM2I128 $0x02, Y7, Y11, Y0 |
| VPERM2I128 $0x02, 224(BP), Y3, Y14 |
| VPERM2I128 $0x13, Y7, Y11, Y12 |
| VPERM2I128 $0x13, 224(BP), Y3, Y4 |
| VPXOR 384(SI), Y0, Y0 |
| VPXOR 416(SI), Y14, Y14 |
| VPXOR 448(SI), Y12, Y12 |
| VPXOR 480(SI), Y4, Y4 |
| VMOVDQU Y0, 384(DI) |
| VMOVDQU Y14, 416(DI) |
| VMOVDQU Y12, 448(DI) |
| VMOVDQU Y4, 480(DI) |
| LEAQ 512(SI), SI |
| SUBQ $0x00000200, BX |
| CMPQ BX, $0x00000200 |
| JG sealAVX2MainLoop |
| |
| // Tail can only hash 480 bytes |
| ADDQ (DI), R10 |
| ADCQ 8(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| ADDQ 16(DI), R10 |
| ADCQ 24(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 32(DI), DI |
| MOVQ $0x0000000a, CX |
| MOVQ $0x00000000, R9 |
| CMPQ BX, $0x80 |
| JBE sealAVX2Tail128 |
| CMPQ BX, $0x00000100 |
| JBE sealAVX2Tail256 |
| CMPQ BX, $0x00000180 |
| JBE sealAVX2Tail384 |
| JMP sealAVX2Tail512 |
| |
| sealSSETail: |
| TESTQ BX, BX |
| JE sealSSEFinalize |
| |
| // We can only load the PT one byte at a time to avoid read after end of buffer |
| MOVQ BX, R9 |
| SHLQ $0x04, R9 |
| LEAQ ·andMask<>+0(SB), R13 |
| MOVQ BX, CX |
| LEAQ -1(SI)(BX*1), SI |
| XORQ R15, R15 |
| XORQ R8, R8 |
| XORQ AX, AX |
| |
| sealSSETailLoadLoop: |
| SHLQ $0x08, R15, R8 |
| SHLQ $0x08, R15 |
| MOVB (SI), AX |
| XORQ AX, R15 |
| LEAQ -1(SI), SI |
| DECQ CX |
| JNE sealSSETailLoadLoop |
| MOVQ R15, 64(BP) |
| MOVQ R8, 72(BP) |
| PXOR 64(BP), X1 |
| MOVOU X1, (DI) |
| MOVOU -16(R13)(R9*1), X12 |
| PAND X12, X1 |
| MOVQ X1, R13 |
| PSRLDQ $0x08, X1 |
| MOVQ X1, R14 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| ADDQ BX, DI |
| |
| sealSSEFinalize: |
| // Hash in the buffer lengths |
| ADDQ ad_len+80(FP), R10 |
| ADCQ src_len+56(FP), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| |
| // Final reduce |
| MOVQ R10, R13 |
| MOVQ R11, R14 |
| MOVQ R12, R15 |
| SUBQ $-5, R10 |
| SBBQ $-1, R11 |
| SBBQ $0x03, R12 |
| CMOVQCS R13, R10 |
| CMOVQCS R14, R11 |
| CMOVQCS R15, R12 |
| |
| // Add in the "s" part of the key |
| ADDQ 16(BP), R10 |
| ADCQ 24(BP), R11 |
| |
| // Finally store the tag at the end of the message |
| MOVQ R10, (DI) |
| MOVQ R11, 8(DI) |
| RET |
| |
| seal192AVX2: |
| VMOVDQA Y0, Y5 |
| VMOVDQA Y14, Y9 |
| VMOVDQA Y12, Y13 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 |
| VMOVDQA Y0, Y6 |
| VMOVDQA Y14, Y10 |
| VMOVDQA Y12, Y8 |
| VMOVDQA Y4, Y2 |
| VMOVDQA Y1, Y15 |
| MOVQ $0x0000000a, R9 |
| |
| sealAVX2192InnerCipherLoop: |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPALIGNR $0x04, Y14, Y14, Y14 |
| VPALIGNR $0x04, Y9, Y9, Y9 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x0c, Y4, Y4, Y4 |
| VPALIGNR $0x0c, Y1, Y1, Y1 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPALIGNR $0x0c, Y14, Y14, Y14 |
| VPALIGNR $0x0c, Y9, Y9, Y9 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x04, Y4, Y4, Y4 |
| VPALIGNR $0x04, Y1, Y1, Y1 |
| DECQ R9 |
| JNE sealAVX2192InnerCipherLoop |
| VPADDD Y6, Y0, Y0 |
| VPADDD Y6, Y5, Y5 |
| VPADDD Y10, Y14, Y14 |
| VPADDD Y10, Y9, Y9 |
| VPADDD Y8, Y12, Y12 |
| VPADDD Y8, Y13, Y13 |
| VPADDD Y2, Y4, Y4 |
| VPADDD Y15, Y1, Y1 |
| VPERM2I128 $0x02, Y0, Y14, Y3 |
| |
| // Clamp and store poly key |
| VPAND ·polyClampMask<>+0(SB), Y3, Y3 |
| VMOVDQA Y3, (BP) |
| |
| // Stream for up to 192 bytes |
| VPERM2I128 $0x13, Y0, Y14, Y0 |
| VPERM2I128 $0x13, Y12, Y4, Y14 |
| VPERM2I128 $0x02, Y5, Y9, Y12 |
| VPERM2I128 $0x02, Y13, Y1, Y4 |
| VPERM2I128 $0x13, Y5, Y9, Y5 |
| VPERM2I128 $0x13, Y13, Y1, Y9 |
| |
| sealAVX2ShortSeal: |
| // Hash aad |
| MOVQ ad_len+80(FP), R9 |
| CALL polyHashADInternal<>(SB) |
| XORQ CX, CX |
| |
| sealAVX2SealHash: |
| // itr1 holds the number of bytes encrypted but not yet hashed |
| CMPQ CX, $0x10 |
| JB sealAVX2ShortSealLoop |
| ADDQ (DI), R10 |
| ADCQ 8(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| SUBQ $0x10, CX |
| ADDQ $0x10, DI |
| JMP sealAVX2SealHash |
| |
| sealAVX2ShortSealLoop: |
| CMPQ BX, $0x20 |
| JB sealAVX2ShortTail32 |
| SUBQ $0x20, BX |
| |
| // Load for encryption |
| VPXOR (SI), Y0, Y0 |
| VMOVDQU Y0, (DI) |
| LEAQ 32(SI), SI |
| |
| // Now can hash |
| ADDQ (DI), R10 |
| ADCQ 8(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| ADDQ 16(DI), R10 |
| ADCQ 24(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 32(DI), DI |
| |
| // Shift stream left |
| VMOVDQA Y14, Y0 |
| VMOVDQA Y12, Y14 |
| VMOVDQA Y4, Y12 |
| VMOVDQA Y5, Y4 |
| VMOVDQA Y9, Y5 |
| VMOVDQA Y13, Y9 |
| VMOVDQA Y1, Y13 |
| VMOVDQA Y6, Y1 |
| VMOVDQA Y10, Y6 |
| JMP sealAVX2ShortSealLoop |
| |
| sealAVX2ShortTail32: |
| CMPQ BX, $0x10 |
| VMOVDQA X0, X1 |
| JB sealAVX2ShortDone |
| SUBQ $0x10, BX |
| |
| // Load for encryption |
| VPXOR (SI), X0, X12 |
| VMOVDQU X12, (DI) |
| LEAQ 16(SI), SI |
| |
| // Hash |
| ADDQ (DI), R10 |
| ADCQ 8(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 16(DI), DI |
| VPERM2I128 $0x11, Y0, Y0, Y0 |
| VMOVDQA X0, X1 |
| |
| sealAVX2ShortDone: |
| VZEROUPPER |
| JMP sealSSETail |
| |
| seal320AVX2: |
| VMOVDQA Y0, Y5 |
| VMOVDQA Y14, Y9 |
| VMOVDQA Y12, Y13 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 |
| VMOVDQA Y0, Y6 |
| VMOVDQA Y14, Y10 |
| VMOVDQA Y12, Y8 |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 |
| VMOVDQA Y14, Y7 |
| VMOVDQA Y12, Y11 |
| VMOVDQA Y4, Y15 |
| MOVQ $0x0000000a, R9 |
| |
| sealAVX2320InnerCipherLoop: |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x0c, Y10, Y3 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y3, Y10, Y10 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x07, Y10, Y3 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y3, Y10, Y10 |
| VPALIGNR $0x04, Y14, Y14, Y14 |
| VPALIGNR $0x04, Y9, Y9, Y9 |
| VPALIGNR $0x04, Y10, Y10, Y10 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x0c, Y4, Y4, Y4 |
| VPALIGNR $0x0c, Y1, Y1, Y1 |
| VPALIGNR $0x0c, Y2, Y2, Y2 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x0c, Y10, Y3 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y3, Y10, Y10 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x07, Y10, Y3 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y3, Y10, Y10 |
| VPALIGNR $0x0c, Y14, Y14, Y14 |
| VPALIGNR $0x0c, Y9, Y9, Y9 |
| VPALIGNR $0x0c, Y10, Y10, Y10 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x04, Y4, Y4, Y4 |
| VPALIGNR $0x04, Y1, Y1, Y1 |
| VPALIGNR $0x04, Y2, Y2, Y2 |
| DECQ R9 |
| JNE sealAVX2320InnerCipherLoop |
| VMOVDQA ·chacha20Constants<>+0(SB), Y3 |
| VPADDD Y3, Y0, Y0 |
| VPADDD Y3, Y5, Y5 |
| VPADDD Y3, Y6, Y6 |
| VPADDD Y7, Y14, Y14 |
| VPADDD Y7, Y9, Y9 |
| VPADDD Y7, Y10, Y10 |
| VPADDD Y11, Y12, Y12 |
| VPADDD Y11, Y13, Y13 |
| VPADDD Y11, Y8, Y8 |
| VMOVDQA ·avx2IncMask<>+0(SB), Y3 |
| VPADDD Y15, Y4, Y4 |
| VPADDD Y3, Y15, Y15 |
| VPADDD Y15, Y1, Y1 |
| VPADDD Y3, Y15, Y15 |
| VPADDD Y15, Y2, Y2 |
| |
| // Clamp and store poly key |
| VPERM2I128 $0x02, Y0, Y14, Y3 |
| VPAND ·polyClampMask<>+0(SB), Y3, Y3 |
| VMOVDQA Y3, (BP) |
| |
| // Stream for up to 320 bytes |
| VPERM2I128 $0x13, Y0, Y14, Y0 |
| VPERM2I128 $0x13, Y12, Y4, Y14 |
| VPERM2I128 $0x02, Y5, Y9, Y12 |
| VPERM2I128 $0x02, Y13, Y1, Y4 |
| VPERM2I128 $0x13, Y5, Y9, Y5 |
| VPERM2I128 $0x13, Y13, Y1, Y9 |
| VPERM2I128 $0x02, Y6, Y10, Y13 |
| VPERM2I128 $0x02, Y8, Y2, Y1 |
| VPERM2I128 $0x13, Y6, Y10, Y6 |
| VPERM2I128 $0x13, Y8, Y2, Y10 |
| JMP sealAVX2ShortSeal |
| |
| sealAVX2Tail128: |
| VMOVDQA ·chacha20Constants<>+0(SB), Y0 |
| VMOVDQA 32(BP), Y14 |
| VMOVDQA 64(BP), Y12 |
| VMOVDQA 192(BP), Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 |
| VMOVDQA Y4, Y1 |
| |
| sealAVX2Tail128LoopA: |
| ADDQ (DI), R10 |
| ADCQ 8(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 16(DI), DI |
| |
| sealAVX2Tail128LoopB: |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| ADDQ (DI), R10 |
| ADCQ 8(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| VPALIGNR $0x04, Y14, Y14, Y14 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x0c, Y4, Y4, Y4 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| ADDQ 16(DI), R10 |
| ADCQ 24(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 32(DI), DI |
| VPALIGNR $0x0c, Y14, Y14, Y14 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x04, Y4, Y4, Y4 |
| DECQ CX |
| JG sealAVX2Tail128LoopA |
| DECQ R9 |
| JGE sealAVX2Tail128LoopB |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y5 |
| VPADDD 32(BP), Y14, Y9 |
| VPADDD 64(BP), Y12, Y13 |
| VPADDD Y1, Y4, Y1 |
| VPERM2I128 $0x02, Y5, Y9, Y0 |
| VPERM2I128 $0x02, Y13, Y1, Y14 |
| VPERM2I128 $0x13, Y5, Y9, Y12 |
| VPERM2I128 $0x13, Y13, Y1, Y4 |
| JMP sealAVX2ShortSealLoop |
| |
| sealAVX2Tail256: |
| VMOVDQA ·chacha20Constants<>+0(SB), Y0 |
| VMOVDQA ·chacha20Constants<>+0(SB), Y5 |
| VMOVDQA 32(BP), Y14 |
| VMOVDQA 32(BP), Y9 |
| VMOVDQA 64(BP), Y12 |
| VMOVDQA 64(BP), Y13 |
| VMOVDQA 192(BP), Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 |
| VMOVDQA Y4, Y7 |
| VMOVDQA Y1, Y11 |
| |
| sealAVX2Tail256LoopA: |
| ADDQ (DI), R10 |
| ADCQ 8(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 16(DI), DI |
| |
| sealAVX2Tail256LoopB: |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| ADDQ (DI), R10 |
| ADCQ 8(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| VPALIGNR $0x04, Y14, Y14, Y14 |
| VPALIGNR $0x04, Y9, Y9, Y9 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x0c, Y4, Y4, Y4 |
| VPALIGNR $0x0c, Y1, Y1, Y1 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| ADDQ 16(DI), R10 |
| ADCQ 24(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 32(DI), DI |
| VPALIGNR $0x0c, Y14, Y14, Y14 |
| VPALIGNR $0x0c, Y9, Y9, Y9 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x04, Y4, Y4, Y4 |
| VPALIGNR $0x04, Y1, Y1, Y1 |
| DECQ CX |
| JG sealAVX2Tail256LoopA |
| DECQ R9 |
| JGE sealAVX2Tail256LoopB |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 |
| VPADDD 32(BP), Y14, Y14 |
| VPADDD 32(BP), Y9, Y9 |
| VPADDD 64(BP), Y12, Y12 |
| VPADDD 64(BP), Y13, Y13 |
| VPADDD Y7, Y4, Y4 |
| VPADDD Y11, Y1, Y1 |
| VPERM2I128 $0x02, Y0, Y14, Y3 |
| VPERM2I128 $0x02, Y12, Y4, Y7 |
| VPERM2I128 $0x13, Y0, Y14, Y11 |
| VPERM2I128 $0x13, Y12, Y4, Y15 |
| VPXOR (SI), Y3, Y3 |
| VPXOR 32(SI), Y7, Y7 |
| VPXOR 64(SI), Y11, Y11 |
| VPXOR 96(SI), Y15, Y15 |
| VMOVDQU Y3, (DI) |
| VMOVDQU Y7, 32(DI) |
| VMOVDQU Y11, 64(DI) |
| VMOVDQU Y15, 96(DI) |
| MOVQ $0x00000080, CX |
| LEAQ 128(SI), SI |
| SUBQ $0x80, BX |
| VPERM2I128 $0x02, Y5, Y9, Y0 |
| VPERM2I128 $0x02, Y13, Y1, Y14 |
| VPERM2I128 $0x13, Y5, Y9, Y12 |
| VPERM2I128 $0x13, Y13, Y1, Y4 |
| JMP sealAVX2SealHash |
| |
| sealAVX2Tail384: |
| VMOVDQA ·chacha20Constants<>+0(SB), Y0 |
| VMOVDQA Y0, Y5 |
| VMOVDQA Y0, Y6 |
| VMOVDQA 32(BP), Y14 |
| VMOVDQA Y14, Y9 |
| VMOVDQA Y14, Y10 |
| VMOVDQA 64(BP), Y12 |
| VMOVDQA Y12, Y13 |
| VMOVDQA Y12, Y8 |
| VMOVDQA 192(BP), Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 |
| VMOVDQA Y4, Y7 |
| VMOVDQA Y1, Y11 |
| VMOVDQA Y2, Y15 |
| |
| sealAVX2Tail384LoopA: |
| ADDQ (DI), R10 |
| ADCQ 8(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 16(DI), DI |
| |
| sealAVX2Tail384LoopB: |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x0c, Y10, Y3 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y3, Y10, Y10 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x07, Y10, Y3 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y3, Y10, Y10 |
| ADDQ (DI), R10 |
| ADCQ 8(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| VPALIGNR $0x04, Y14, Y14, Y14 |
| VPALIGNR $0x04, Y9, Y9, Y9 |
| VPALIGNR $0x04, Y10, Y10, Y10 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x0c, Y4, Y4, Y4 |
| VPALIGNR $0x0c, Y1, Y1, Y1 |
| VPALIGNR $0x0c, Y2, Y2, Y2 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x0c, Y14, Y3 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y14, Y0, Y0 |
| VPXOR Y0, Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPADDD Y4, Y12, Y12 |
| VPXOR Y12, Y14, Y14 |
| VPSLLD $0x07, Y14, Y3 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y3, Y14, Y14 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x0c, Y9, Y3 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y9, Y5, Y5 |
| VPXOR Y5, Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPADDD Y1, Y13, Y13 |
| VPXOR Y13, Y9, Y9 |
| VPSLLD $0x07, Y9, Y3 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y3, Y9, Y9 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x0c, Y10, Y3 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y3, Y10, Y10 |
| VPADDD Y10, Y6, Y6 |
| VPXOR Y6, Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPADDD Y2, Y8, Y8 |
| VPXOR Y8, Y10, Y10 |
| VPSLLD $0x07, Y10, Y3 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y3, Y10, Y10 |
| ADDQ 16(DI), R10 |
| ADCQ 24(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 32(DI), DI |
| VPALIGNR $0x0c, Y14, Y14, Y14 |
| VPALIGNR $0x0c, Y9, Y9, Y9 |
| VPALIGNR $0x0c, Y10, Y10, Y10 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x04, Y4, Y4, Y4 |
| VPALIGNR $0x04, Y1, Y1, Y1 |
| VPALIGNR $0x04, Y2, Y2, Y2 |
| DECQ CX |
| JG sealAVX2Tail384LoopA |
| DECQ R9 |
| JGE sealAVX2Tail384LoopB |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 |
| VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 |
| VPADDD 32(BP), Y14, Y14 |
| VPADDD 32(BP), Y9, Y9 |
| VPADDD 32(BP), Y10, Y10 |
| VPADDD 64(BP), Y12, Y12 |
| VPADDD 64(BP), Y13, Y13 |
| VPADDD 64(BP), Y8, Y8 |
| VPADDD Y7, Y4, Y4 |
| VPADDD Y11, Y1, Y1 |
| VPADDD Y15, Y2, Y2 |
| VPERM2I128 $0x02, Y0, Y14, Y3 |
| VPERM2I128 $0x02, Y12, Y4, Y7 |
| VPERM2I128 $0x13, Y0, Y14, Y11 |
| VPERM2I128 $0x13, Y12, Y4, Y15 |
| VPXOR (SI), Y3, Y3 |
| VPXOR 32(SI), Y7, Y7 |
| VPXOR 64(SI), Y11, Y11 |
| VPXOR 96(SI), Y15, Y15 |
| VMOVDQU Y3, (DI) |
| VMOVDQU Y7, 32(DI) |
| VMOVDQU Y11, 64(DI) |
| VMOVDQU Y15, 96(DI) |
| VPERM2I128 $0x02, Y5, Y9, Y3 |
| VPERM2I128 $0x02, Y13, Y1, Y7 |
| VPERM2I128 $0x13, Y5, Y9, Y11 |
| VPERM2I128 $0x13, Y13, Y1, Y15 |
| VPXOR 128(SI), Y3, Y3 |
| VPXOR 160(SI), Y7, Y7 |
| VPXOR 192(SI), Y11, Y11 |
| VPXOR 224(SI), Y15, Y15 |
| VMOVDQU Y3, 128(DI) |
| VMOVDQU Y7, 160(DI) |
| VMOVDQU Y11, 192(DI) |
| VMOVDQU Y15, 224(DI) |
| MOVQ $0x00000100, CX |
| LEAQ 256(SI), SI |
| SUBQ $0x00000100, BX |
| VPERM2I128 $0x02, Y6, Y10, Y0 |
| VPERM2I128 $0x02, Y8, Y2, Y14 |
| VPERM2I128 $0x13, Y6, Y10, Y12 |
| VPERM2I128 $0x13, Y8, Y2, Y4 |
| JMP sealAVX2SealHash |
| |
| sealAVX2Tail512: |
| VMOVDQA ·chacha20Constants<>+0(SB), Y0 |
| VMOVDQA Y0, Y5 |
| VMOVDQA Y0, Y6 |
| VMOVDQA Y0, Y7 |
| VMOVDQA 32(BP), Y14 |
| VMOVDQA Y14, Y9 |
| VMOVDQA Y14, Y10 |
| VMOVDQA Y14, Y11 |
| VMOVDQA 64(BP), Y12 |
| VMOVDQA Y12, Y13 |
| VMOVDQA Y12, Y8 |
| VMOVDQA Y12, Y15 |
| VMOVDQA 192(BP), Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 |
| VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 |
| VMOVDQA Y4, 96(BP) |
| VMOVDQA Y1, 128(BP) |
| VMOVDQA Y2, 160(BP) |
| VMOVDQA Y3, 192(BP) |
| |
| sealAVX2Tail512LoopA: |
| ADDQ (DI), R10 |
| ADCQ 8(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), AX |
| MOVQ AX, R15 |
| MULQ R10 |
| MOVQ AX, R13 |
| MOVQ DX, R14 |
| MOVQ (BP), AX |
| MULQ R11 |
| IMULQ R12, R15 |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), AX |
| MOVQ AX, R8 |
| MULQ R10 |
| ADDQ AX, R14 |
| ADCQ $0x00, DX |
| MOVQ DX, R10 |
| MOVQ 8(BP), AX |
| MULQ R11 |
| ADDQ AX, R15 |
| ADCQ $0x00, DX |
| IMULQ R12, R8 |
| ADDQ R10, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 16(DI), DI |
| |
| sealAVX2Tail512LoopB: |
| VPADDD Y14, Y0, Y0 |
| VPADDD Y9, Y5, Y5 |
| VPADDD Y10, Y6, Y6 |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y0, Y4, Y4 |
| VPXOR Y5, Y1, Y1 |
| VPXOR Y6, Y2, Y2 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 |
| VPADDD Y4, Y12, Y12 |
| VPADDD Y1, Y13, Y13 |
| VPADDD Y2, Y8, Y8 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y12, Y14, Y14 |
| VPXOR Y13, Y9, Y9 |
| VPXOR Y8, Y10, Y10 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA Y15, 224(BP) |
| VPSLLD $0x0c, Y14, Y15 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPSLLD $0x0c, Y9, Y15 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPSLLD $0x0c, Y10, Y15 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPSLLD $0x0c, Y11, Y15 |
| VPSRLD $0x14, Y11, Y11 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA 224(BP), Y15 |
| ADDQ (DI), R10 |
| ADCQ 8(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| VPADDD Y14, Y0, Y0 |
| VPADDD Y9, Y5, Y5 |
| VPADDD Y10, Y6, Y6 |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y0, Y4, Y4 |
| VPXOR Y5, Y1, Y1 |
| VPXOR Y6, Y2, Y2 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 |
| VPADDD Y4, Y12, Y12 |
| VPADDD Y1, Y13, Y13 |
| VPADDD Y2, Y8, Y8 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y12, Y14, Y14 |
| VPXOR Y13, Y9, Y9 |
| VPXOR Y8, Y10, Y10 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA Y15, 224(BP) |
| VPSLLD $0x07, Y14, Y15 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPSLLD $0x07, Y9, Y15 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPSLLD $0x07, Y10, Y15 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPSLLD $0x07, Y11, Y15 |
| VPSRLD $0x19, Y11, Y11 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA 224(BP), Y15 |
| VPALIGNR $0x04, Y14, Y14, Y14 |
| VPALIGNR $0x04, Y9, Y9, Y9 |
| VPALIGNR $0x04, Y10, Y10, Y10 |
| VPALIGNR $0x04, Y11, Y11, Y11 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x08, Y15, Y15, Y15 |
| VPALIGNR $0x0c, Y4, Y4, Y4 |
| VPALIGNR $0x0c, Y1, Y1, Y1 |
| VPALIGNR $0x0c, Y2, Y2, Y2 |
| VPALIGNR $0x0c, Y3, Y3, Y3 |
| VPADDD Y14, Y0, Y0 |
| VPADDD Y9, Y5, Y5 |
| VPADDD Y10, Y6, Y6 |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y0, Y4, Y4 |
| VPXOR Y5, Y1, Y1 |
| VPXOR Y6, Y2, Y2 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 |
| VPADDD Y4, Y12, Y12 |
| VPADDD Y1, Y13, Y13 |
| VPADDD Y2, Y8, Y8 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y12, Y14, Y14 |
| VPXOR Y13, Y9, Y9 |
| VPXOR Y8, Y10, Y10 |
| VPXOR Y15, Y11, Y11 |
| ADDQ 16(DI), R10 |
| ADCQ 24(DI), R11 |
| ADCQ $0x01, R12 |
| MOVQ (BP), DX |
| MOVQ DX, R15 |
| MULXQ R10, R13, R14 |
| IMULQ R12, R15 |
| MULXQ R11, AX, DX |
| ADDQ AX, R14 |
| ADCQ DX, R15 |
| MOVQ 8(BP), DX |
| MULXQ R10, R10, AX |
| ADDQ R10, R14 |
| MULXQ R11, R11, R8 |
| ADCQ R11, R15 |
| ADCQ $0x00, R8 |
| IMULQ R12, DX |
| ADDQ AX, R15 |
| ADCQ DX, R8 |
| MOVQ R13, R10 |
| MOVQ R14, R11 |
| MOVQ R15, R12 |
| ANDQ $0x03, R12 |
| MOVQ R15, R13 |
| ANDQ $-4, R13 |
| MOVQ R8, R14 |
| SHRQ $0x02, R8, R15 |
| SHRQ $0x02, R8 |
| ADDQ R13, R10 |
| ADCQ R14, R11 |
| ADCQ $0x00, R12 |
| ADDQ R15, R10 |
| ADCQ R8, R11 |
| ADCQ $0x00, R12 |
| LEAQ 32(DI), DI |
| VMOVDQA Y15, 224(BP) |
| VPSLLD $0x0c, Y14, Y15 |
| VPSRLD $0x14, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPSLLD $0x0c, Y9, Y15 |
| VPSRLD $0x14, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPSLLD $0x0c, Y10, Y15 |
| VPSRLD $0x14, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPSLLD $0x0c, Y11, Y15 |
| VPSRLD $0x14, Y11, Y11 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA 224(BP), Y15 |
| VPADDD Y14, Y0, Y0 |
| VPADDD Y9, Y5, Y5 |
| VPADDD Y10, Y6, Y6 |
| VPADDD Y11, Y7, Y7 |
| VPXOR Y0, Y4, Y4 |
| VPXOR Y5, Y1, Y1 |
| VPXOR Y6, Y2, Y2 |
| VPXOR Y7, Y3, Y3 |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 |
| VPADDD Y4, Y12, Y12 |
| VPADDD Y1, Y13, Y13 |
| VPADDD Y2, Y8, Y8 |
| VPADDD Y3, Y15, Y15 |
| VPXOR Y12, Y14, Y14 |
| VPXOR Y13, Y9, Y9 |
| VPXOR Y8, Y10, Y10 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA Y15, 224(BP) |
| VPSLLD $0x07, Y14, Y15 |
| VPSRLD $0x19, Y14, Y14 |
| VPXOR Y15, Y14, Y14 |
| VPSLLD $0x07, Y9, Y15 |
| VPSRLD $0x19, Y9, Y9 |
| VPXOR Y15, Y9, Y9 |
| VPSLLD $0x07, Y10, Y15 |
| VPSRLD $0x19, Y10, Y10 |
| VPXOR Y15, Y10, Y10 |
| VPSLLD $0x07, Y11, Y15 |
| VPSRLD $0x19, Y11, Y11 |
| VPXOR Y15, Y11, Y11 |
| VMOVDQA 224(BP), Y15 |
| VPALIGNR $0x0c, Y14, Y14, Y14 |
| VPALIGNR $0x0c, Y9, Y9, Y9 |
| VPALIGNR $0x0c, Y10, Y10, Y10 |
| VPALIGNR $0x0c, Y11, Y11, Y11 |
| VPALIGNR $0x08, Y12, Y12, Y12 |
| VPALIGNR $0x08, Y13, Y13, Y13 |
| VPALIGNR $0x08, Y8, Y8, Y8 |
| VPALIGNR $0x08, Y15, Y15, Y15 |
| VPALIGNR $0x04, Y4, Y4, Y4 |
| VPALIGNR $0x04, Y1, Y1, Y1 |
| VPALIGNR $0x04, Y2, Y2, Y2 |
| VPALIGNR $0x04, Y3, Y3, Y3 |
| DECQ CX |
| JG sealAVX2Tail512LoopA |
| DECQ R9 |
| JGE sealAVX2Tail512LoopB |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 |
| VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 |
| VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 |
| VPADDD 32(BP), Y14, Y14 |
| VPADDD 32(BP), Y9, Y9 |
| VPADDD 32(BP), Y10, Y10 |
| VPADDD 32(BP), Y11, Y11 |
| VPADDD 64(BP), Y12, Y12 |
| VPADDD 64(BP), Y13, Y13 |
| VPADDD 64(BP), Y8, Y8 |
| VPADDD 64(BP), Y15, Y15 |
| VPADDD 96(BP), Y4, Y4 |
| VPADDD 128(BP), Y1, Y1 |
| VPADDD 160(BP), Y2, Y2 |
| VPADDD 192(BP), Y3, Y3 |
| VMOVDQA Y15, 224(BP) |
| VPERM2I128 $0x02, Y0, Y14, Y15 |
| VPXOR (SI), Y15, Y15 |
| VMOVDQU Y15, (DI) |
| VPERM2I128 $0x02, Y12, Y4, Y15 |
| VPXOR 32(SI), Y15, Y15 |
| VMOVDQU Y15, 32(DI) |
| VPERM2I128 $0x13, Y0, Y14, Y15 |
| VPXOR 64(SI), Y15, Y15 |
| VMOVDQU Y15, 64(DI) |
| VPERM2I128 $0x13, Y12, Y4, Y15 |
| VPXOR 96(SI), Y15, Y15 |
| VMOVDQU Y15, 96(DI) |
| VPERM2I128 $0x02, Y5, Y9, Y0 |
| VPERM2I128 $0x02, Y13, Y1, Y14 |
| VPERM2I128 $0x13, Y5, Y9, Y12 |
| VPERM2I128 $0x13, Y13, Y1, Y4 |
| VPXOR 128(SI), Y0, Y0 |
| VPXOR 160(SI), Y14, Y14 |
| VPXOR 192(SI), Y12, Y12 |
| VPXOR 224(SI), Y4, Y4 |
| VMOVDQU Y0, 128(DI) |
| VMOVDQU Y14, 160(DI) |
| VMOVDQU Y12, 192(DI) |
| VMOVDQU Y4, 224(DI) |
| VPERM2I128 $0x02, Y6, Y10, Y0 |
| VPERM2I128 $0x02, Y8, Y2, Y14 |
| VPERM2I128 $0x13, Y6, Y10, Y12 |
| VPERM2I128 $0x13, Y8, Y2, Y4 |
| VPXOR 256(SI), Y0, Y0 |
| VPXOR 288(SI), Y14, Y14 |
| VPXOR 320(SI), Y12, Y12 |
| VPXOR 352(SI), Y4, Y4 |
| VMOVDQU Y0, 256(DI) |
| VMOVDQU Y14, 288(DI) |
| VMOVDQU Y12, 320(DI) |
| VMOVDQU Y4, 352(DI) |
| MOVQ $0x00000180, CX |
| LEAQ 384(SI), SI |
| SUBQ $0x00000180, BX |
| VPERM2I128 $0x02, Y7, Y11, Y0 |
| VPERM2I128 $0x02, 224(BP), Y3, Y14 |
| VPERM2I128 $0x13, Y7, Y11, Y12 |
| VPERM2I128 $0x13, 224(BP), Y3, Y4 |
| JMP sealAVX2SealHash |
