| // Code generated by command: go run chacha20poly1305_amd64_asm.go -out ../chacha20poly1305_amd64.s -pkg chacha20poly1305. DO NOT EDIT. | |
| //go:build gc && !purego | |
| #include "textflag.h" | |
| // func polyHashADInternal<>() | |
| TEXT polyHashADInternal<>(SB), NOSPLIT, $0 | |
| XORQ R10, R10 | |
| XORQ R11, R11 | |
| XORQ R12, R12 | |
| CMPQ R9, $0x0d | |
| JNE hashADLoop | |
| MOVQ (CX), R10 | |
| MOVQ 5(CX), R11 | |
| SHRQ $0x18, R11 | |
| MOVQ $0x00000001, R12 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| RET | |
| hashADLoop: | |
| // Hash in 16 byte chunks | |
| CMPQ R9, $0x10 | |
| JB hashADTail | |
| ADDQ (CX), R10 | |
| ADCQ 8(CX), R11 | |
| ADCQ $0x01, R12 | |
| LEAQ 16(CX), CX | |
| SUBQ $0x10, R9 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| JMP hashADLoop | |
| hashADTail: | |
| CMPQ R9, $0x00 | |
| JE hashADDone | |
| // Hash last < 16 byte tail | |
| XORQ R13, R13 | |
| XORQ R14, R14 | |
| XORQ R15, R15 | |
| ADDQ R9, CX | |
| hashADTailLoop: | |
| SHLQ $0x08, R13, R14 | |
| SHLQ $0x08, R13 | |
| MOVB -1(CX), R15 | |
| XORQ R15, R13 | |
| DECQ CX | |
| DECQ R9 | |
| JNE hashADTailLoop | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| hashADDone: | |
| RET | |
| // func chacha20Poly1305Open(dst []byte, key []uint32, src []byte, ad []byte) bool | |
| // Requires: AVX, AVX2, BMI2, CMOV, SSE2 | |
| TEXT ·chacha20Poly1305Open(SB), $288-97 | |
| // For aligned stack access | |
| MOVQ SP, BP | |
| ADDQ $0x20, BP | |
| ANDQ $-32, BP | |
| MOVQ dst_base+0(FP), DI | |
| MOVQ key_base+24(FP), R8 | |
| MOVQ src_base+48(FP), SI | |
| MOVQ src_len+56(FP), BX | |
| MOVQ ad_base+72(FP), CX | |
| VZEROUPPER | |
| VMOVDQU ·chacha20Constants<>+0(SB), Y0 | |
| VBROADCASTI128 16(R8), Y14 | |
| VBROADCASTI128 32(R8), Y12 | |
| VBROADCASTI128 48(R8), Y4 | |
| VPADDD ·avx2InitMask<>+0(SB), Y4, Y4 | |
| // Special optimization, for very short buffers | |
| CMPQ BX, $0xc0 | |
| JBE openAVX2192 | |
| CMPQ BX, $0x00000140 | |
| JBE openAVX2320 | |
| // For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream | |
| VMOVDQA Y14, 32(BP) | |
| VMOVDQA Y12, 64(BP) | |
| VMOVDQA Y4, 192(BP) | |
| MOVQ $0x0000000a, R9 | |
| openAVX2PreparePolyKey: | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPALIGNR $0x04, Y14, Y14, Y14 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x0c, Y4, Y4, Y4 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPALIGNR $0x0c, Y14, Y14, Y14 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x04, Y4, Y4, Y4 | |
| DECQ R9 | |
| JNE openAVX2PreparePolyKey | |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 | |
| VPADDD 32(BP), Y14, Y14 | |
| VPADDD 64(BP), Y12, Y12 | |
| VPADDD 192(BP), Y4, Y4 | |
| VPERM2I128 $0x02, Y0, Y14, Y3 | |
| // Clamp and store poly key | |
| VPAND ·polyClampMask<>+0(SB), Y3, Y3 | |
| VMOVDQA Y3, (BP) | |
| // Stream for the first 64 bytes | |
| VPERM2I128 $0x13, Y0, Y14, Y0 | |
| VPERM2I128 $0x13, Y12, Y4, Y14 | |
| // Hash AD + first 64 bytes | |
| MOVQ ad_len+80(FP), R9 | |
| CALL polyHashADInternal<>(SB) | |
| XORQ CX, CX | |
| openAVX2InitialHash64: | |
| ADDQ (SI)(CX*1), R10 | |
| ADCQ 8(SI)(CX*1), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ $0x10, CX | |
| CMPQ CX, $0x40 | |
| JNE openAVX2InitialHash64 | |
| // Decrypt the first 64 bytes | |
| VPXOR (SI), Y0, Y0 | |
| VPXOR 32(SI), Y14, Y14 | |
| VMOVDQU Y0, (DI) | |
| VMOVDQU Y14, 32(DI) | |
| LEAQ 64(SI), SI | |
| LEAQ 64(DI), DI | |
| SUBQ $0x40, BX | |
| openAVX2MainLoop: | |
| CMPQ BX, $0x00000200 | |
| JB openAVX2MainLoopDone | |
| // Load state, increment counter blocks, store the incremented counters | |
| VMOVDQU ·chacha20Constants<>+0(SB), Y0 | |
| VMOVDQA Y0, Y5 | |
| VMOVDQA Y0, Y6 | |
| VMOVDQA Y0, Y7 | |
| VMOVDQA 32(BP), Y14 | |
| VMOVDQA Y14, Y9 | |
| VMOVDQA Y14, Y10 | |
| VMOVDQA Y14, Y11 | |
| VMOVDQA 64(BP), Y12 | |
| VMOVDQA Y12, Y13 | |
| VMOVDQA Y12, Y8 | |
| VMOVDQA Y12, Y15 | |
| VMOVDQA 192(BP), Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 | |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 | |
| VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 | |
| VMOVDQA Y4, 96(BP) | |
| VMOVDQA Y1, 128(BP) | |
| VMOVDQA Y2, 160(BP) | |
| VMOVDQA Y3, 192(BP) | |
| XORQ CX, CX | |
| openAVX2InternalLoop: | |
| ADDQ (SI)(CX*1), R10 | |
| ADCQ 8(SI)(CX*1), R11 | |
| ADCQ $0x01, R12 | |
| VPADDD Y14, Y0, Y0 | |
| VPADDD Y9, Y5, Y5 | |
| VPADDD Y10, Y6, Y6 | |
| VPADDD Y11, Y7, Y7 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| VPXOR Y0, Y4, Y4 | |
| VPXOR Y5, Y1, Y1 | |
| VPXOR Y6, Y2, Y2 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| VPADDD Y4, Y12, Y12 | |
| VPADDD Y1, Y13, Y13 | |
| VPADDD Y2, Y8, Y8 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y12, Y14, Y14 | |
| VPXOR Y13, Y9, Y9 | |
| VPXOR Y8, Y10, Y10 | |
| VPXOR Y15, Y11, Y11 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| VMOVDQA Y15, 224(BP) | |
| VPSLLD $0x0c, Y14, Y15 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPSLLD $0x0c, Y9, Y15 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPSLLD $0x0c, Y10, Y15 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPSLLD $0x0c, Y11, Y15 | |
| VPSRLD $0x14, Y11, Y11 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA 224(BP), Y15 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| VPADDD Y14, Y0, Y0 | |
| VPADDD Y9, Y5, Y5 | |
| VPADDD Y10, Y6, Y6 | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y0, Y4, Y4 | |
| VPXOR Y5, Y1, Y1 | |
| VPXOR Y6, Y2, Y2 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 | |
| ADDQ 16(SI)(CX*1), R10 | |
| ADCQ 24(SI)(CX*1), R11 | |
| ADCQ $0x01, R12 | |
| VPADDD Y4, Y12, Y12 | |
| VPADDD Y1, Y13, Y13 | |
| VPADDD Y2, Y8, Y8 | |
| VPADDD Y3, Y15, Y15 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| VPXOR Y12, Y14, Y14 | |
| VPXOR Y13, Y9, Y9 | |
| VPXOR Y8, Y10, Y10 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA Y15, 224(BP) | |
| VPSLLD $0x07, Y14, Y15 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPSLLD $0x07, Y9, Y15 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPSLLD $0x07, Y10, Y15 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPSLLD $0x07, Y11, Y15 | |
| VPSRLD $0x19, Y11, Y11 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA 224(BP), Y15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| VPALIGNR $0x04, Y14, Y14, Y14 | |
| VPALIGNR $0x04, Y9, Y9, Y9 | |
| VPALIGNR $0x04, Y10, Y10, Y10 | |
| VPALIGNR $0x04, Y11, Y11, Y11 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x08, Y15, Y15, Y15 | |
| VPALIGNR $0x0c, Y4, Y4, Y4 | |
| VPALIGNR $0x0c, Y1, Y1, Y1 | |
| VPALIGNR $0x0c, Y2, Y2, Y2 | |
| VPALIGNR $0x0c, Y3, Y3, Y3 | |
| VPADDD Y14, Y0, Y0 | |
| VPADDD Y9, Y5, Y5 | |
| VPADDD Y10, Y6, Y6 | |
| VPADDD Y11, Y7, Y7 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| VPXOR Y0, Y4, Y4 | |
| VPXOR Y5, Y1, Y1 | |
| VPXOR Y6, Y2, Y2 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| VPADDD Y4, Y12, Y12 | |
| VPADDD Y1, Y13, Y13 | |
| VPADDD Y2, Y8, Y8 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y12, Y14, Y14 | |
| VPXOR Y13, Y9, Y9 | |
| VPXOR Y8, Y10, Y10 | |
| VPXOR Y15, Y11, Y11 | |
| ADDQ 32(SI)(CX*1), R10 | |
| ADCQ 40(SI)(CX*1), R11 | |
| ADCQ $0x01, R12 | |
| LEAQ 48(CX), CX | |
| VMOVDQA Y15, 224(BP) | |
| VPSLLD $0x0c, Y14, Y15 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPSLLD $0x0c, Y9, Y15 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPSLLD $0x0c, Y10, Y15 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPSLLD $0x0c, Y11, Y15 | |
| VPSRLD $0x14, Y11, Y11 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA 224(BP), Y15 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| VPADDD Y14, Y0, Y0 | |
| VPADDD Y9, Y5, Y5 | |
| VPADDD Y10, Y6, Y6 | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y0, Y4, Y4 | |
| VPXOR Y5, Y1, Y1 | |
| VPXOR Y6, Y2, Y2 | |
| VPXOR Y7, Y3, Y3 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 | |
| VPADDD Y4, Y12, Y12 | |
| VPADDD Y1, Y13, Y13 | |
| VPADDD Y2, Y8, Y8 | |
| VPADDD Y3, Y15, Y15 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| VPXOR Y12, Y14, Y14 | |
| VPXOR Y13, Y9, Y9 | |
| VPXOR Y8, Y10, Y10 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA Y15, 224(BP) | |
| VPSLLD $0x07, Y14, Y15 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPSLLD $0x07, Y9, Y15 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPSLLD $0x07, Y10, Y15 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPSLLD $0x07, Y11, Y15 | |
| VPSRLD $0x19, Y11, Y11 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA 224(BP), Y15 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| VPALIGNR $0x0c, Y14, Y14, Y14 | |
| VPALIGNR $0x0c, Y9, Y9, Y9 | |
| VPALIGNR $0x0c, Y10, Y10, Y10 | |
| VPALIGNR $0x0c, Y11, Y11, Y11 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x08, Y15, Y15, Y15 | |
| VPALIGNR $0x04, Y4, Y4, Y4 | |
| VPALIGNR $0x04, Y1, Y1, Y1 | |
| VPALIGNR $0x04, Y2, Y2, Y2 | |
| VPALIGNR $0x04, Y3, Y3, Y3 | |
| CMPQ CX, $0x000001e0 | |
| JNE openAVX2InternalLoop | |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 | |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 | |
| VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 | |
| VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 | |
| VPADDD 32(BP), Y14, Y14 | |
| VPADDD 32(BP), Y9, Y9 | |
| VPADDD 32(BP), Y10, Y10 | |
| VPADDD 32(BP), Y11, Y11 | |
| VPADDD 64(BP), Y12, Y12 | |
| VPADDD 64(BP), Y13, Y13 | |
| VPADDD 64(BP), Y8, Y8 | |
| VPADDD 64(BP), Y15, Y15 | |
| VPADDD 96(BP), Y4, Y4 | |
| VPADDD 128(BP), Y1, Y1 | |
| VPADDD 160(BP), Y2, Y2 | |
| VPADDD 192(BP), Y3, Y3 | |
| VMOVDQA Y15, 224(BP) | |
| // We only hashed 480 of the 512 bytes available - hash the remaining 32 here | |
| ADDQ 480(SI), R10 | |
| ADCQ 488(SI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| VPERM2I128 $0x02, Y0, Y14, Y15 | |
| VPERM2I128 $0x13, Y0, Y14, Y14 | |
| VPERM2I128 $0x02, Y12, Y4, Y0 | |
| VPERM2I128 $0x13, Y12, Y4, Y12 | |
| VPXOR (SI), Y15, Y15 | |
| VPXOR 32(SI), Y0, Y0 | |
| VPXOR 64(SI), Y14, Y14 | |
| VPXOR 96(SI), Y12, Y12 | |
| VMOVDQU Y15, (DI) | |
| VMOVDQU Y0, 32(DI) | |
| VMOVDQU Y14, 64(DI) | |
| VMOVDQU Y12, 96(DI) | |
| VPERM2I128 $0x02, Y5, Y9, Y0 | |
| VPERM2I128 $0x02, Y13, Y1, Y14 | |
| VPERM2I128 $0x13, Y5, Y9, Y12 | |
| VPERM2I128 $0x13, Y13, Y1, Y4 | |
| VPXOR 128(SI), Y0, Y0 | |
| VPXOR 160(SI), Y14, Y14 | |
| VPXOR 192(SI), Y12, Y12 | |
| VPXOR 224(SI), Y4, Y4 | |
| VMOVDQU Y0, 128(DI) | |
| VMOVDQU Y14, 160(DI) | |
| VMOVDQU Y12, 192(DI) | |
| VMOVDQU Y4, 224(DI) | |
| // and here | |
| ADDQ 496(SI), R10 | |
| ADCQ 504(SI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| VPERM2I128 $0x02, Y6, Y10, Y0 | |
| VPERM2I128 $0x02, Y8, Y2, Y14 | |
| VPERM2I128 $0x13, Y6, Y10, Y12 | |
| VPERM2I128 $0x13, Y8, Y2, Y4 | |
| VPXOR 256(SI), Y0, Y0 | |
| VPXOR 288(SI), Y14, Y14 | |
| VPXOR 320(SI), Y12, Y12 | |
| VPXOR 352(SI), Y4, Y4 | |
| VMOVDQU Y0, 256(DI) | |
| VMOVDQU Y14, 288(DI) | |
| VMOVDQU Y12, 320(DI) | |
| VMOVDQU Y4, 352(DI) | |
| VPERM2I128 $0x02, Y7, Y11, Y0 | |
| VPERM2I128 $0x02, 224(BP), Y3, Y14 | |
| VPERM2I128 $0x13, Y7, Y11, Y12 | |
| VPERM2I128 $0x13, 224(BP), Y3, Y4 | |
| VPXOR 384(SI), Y0, Y0 | |
| VPXOR 416(SI), Y14, Y14 | |
| VPXOR 448(SI), Y12, Y12 | |
| VPXOR 480(SI), Y4, Y4 | |
| VMOVDQU Y0, 384(DI) | |
| VMOVDQU Y14, 416(DI) | |
| VMOVDQU Y12, 448(DI) | |
| VMOVDQU Y4, 480(DI) | |
| LEAQ 512(SI), SI | |
| LEAQ 512(DI), DI | |
| SUBQ $0x00000200, BX | |
| JMP openAVX2MainLoop | |
| openAVX2MainLoopDone: | |
| // Handle the various tail sizes efficiently | |
| TESTQ BX, BX | |
| JE openSSEFinalize | |
| CMPQ BX, $0x80 | |
| JBE openAVX2Tail128 | |
| CMPQ BX, $0x00000100 | |
| JBE openAVX2Tail256 | |
| CMPQ BX, $0x00000180 | |
| JBE openAVX2Tail384 | |
| JMP openAVX2Tail512 | |
| openSSEFinalize: | |
| // Hash in the PT, AAD lengths | |
| ADDQ ad_len+80(FP), R10 | |
| ADCQ src_len+56(FP), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| // Final reduce | |
| MOVQ R10, R13 | |
| MOVQ R11, R14 | |
| MOVQ R12, R15 | |
| SUBQ $-5, R10 | |
| SBBQ $-1, R11 | |
| SBBQ $0x03, R12 | |
| CMOVQCS R13, R10 | |
| CMOVQCS R14, R11 | |
| CMOVQCS R15, R12 | |
| // Add in the "s" part of the key | |
| ADDQ 16(BP), R10 | |
| ADCQ 24(BP), R11 | |
| // Finally, constant time compare to the tag at the end of the message | |
| XORQ AX, AX | |
| MOVQ $0x00000001, DX | |
| XORQ (SI), R10 | |
| XORQ 8(SI), R11 | |
| ORQ R11, R10 | |
| CMOVQEQ DX, AX | |
| // Return true iff tags are equal | |
| MOVB AX, ret+96(FP) | |
| RET | |
| openSSETail16: | |
| TESTQ BX, BX | |
| JE openSSEFinalize | |
| // We can safely load the CT from the end, because it is padded with the MAC | |
| MOVQ BX, R9 | |
| SHLQ $0x04, R9 | |
| LEAQ ·andMask<>+0(SB), R13 | |
| MOVOU (SI), X12 | |
| ADDQ BX, SI | |
| PAND -16(R13)(R9*1), X12 | |
| MOVO X12, 64(BP) | |
| MOVQ X12, R13 | |
| MOVQ 72(BP), R14 | |
| PXOR X1, X12 | |
| // We can only store one byte at a time, since plaintext can be shorter than 16 bytes | |
| openSSETail16Store: | |
| MOVQ X12, R8 | |
| MOVB R8, (DI) | |
| PSRLDQ $0x01, X12 | |
| INCQ DI | |
| DECQ BX | |
| JNE openSSETail16Store | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| JMP openSSEFinalize | |
| openAVX2192: | |
| VMOVDQA Y0, Y5 | |
| VMOVDQA Y14, Y9 | |
| VMOVDQA Y12, Y13 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 | |
| VMOVDQA Y0, Y6 | |
| VMOVDQA Y14, Y10 | |
| VMOVDQA Y12, Y8 | |
| VMOVDQA Y4, Y2 | |
| VMOVDQA Y1, Y15 | |
| MOVQ $0x0000000a, R9 | |
| openAVX2192InnerCipherLoop: | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPALIGNR $0x04, Y14, Y14, Y14 | |
| VPALIGNR $0x04, Y9, Y9, Y9 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x0c, Y4, Y4, Y4 | |
| VPALIGNR $0x0c, Y1, Y1, Y1 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPALIGNR $0x0c, Y14, Y14, Y14 | |
| VPALIGNR $0x0c, Y9, Y9, Y9 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x04, Y4, Y4, Y4 | |
| VPALIGNR $0x04, Y1, Y1, Y1 | |
| DECQ R9 | |
| JNE openAVX2192InnerCipherLoop | |
| VPADDD Y6, Y0, Y0 | |
| VPADDD Y6, Y5, Y5 | |
| VPADDD Y10, Y14, Y14 | |
| VPADDD Y10, Y9, Y9 | |
| VPADDD Y8, Y12, Y12 | |
| VPADDD Y8, Y13, Y13 | |
| VPADDD Y2, Y4, Y4 | |
| VPADDD Y15, Y1, Y1 | |
| VPERM2I128 $0x02, Y0, Y14, Y3 | |
| // Clamp and store poly key | |
| VPAND ·polyClampMask<>+0(SB), Y3, Y3 | |
| VMOVDQA Y3, (BP) | |
| // Stream for up to 192 bytes | |
| VPERM2I128 $0x13, Y0, Y14, Y0 | |
| VPERM2I128 $0x13, Y12, Y4, Y14 | |
| VPERM2I128 $0x02, Y5, Y9, Y12 | |
| VPERM2I128 $0x02, Y13, Y1, Y4 | |
| VPERM2I128 $0x13, Y5, Y9, Y5 | |
| VPERM2I128 $0x13, Y13, Y1, Y9 | |
| openAVX2ShortOpen: | |
| // Hash | |
| MOVQ ad_len+80(FP), R9 | |
| CALL polyHashADInternal<>(SB) | |
| openAVX2ShortOpenLoop: | |
| CMPQ BX, $0x20 | |
| JB openAVX2ShortTail32 | |
| SUBQ $0x20, BX | |
| // Load for hashing | |
| ADDQ (SI), R10 | |
| ADCQ 8(SI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ 16(SI), R10 | |
| ADCQ 24(SI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| // Load for decryption | |
| VPXOR (SI), Y0, Y0 | |
| VMOVDQU Y0, (DI) | |
| LEAQ 32(SI), SI | |
| LEAQ 32(DI), DI | |
| // Shift stream left | |
| VMOVDQA Y14, Y0 | |
| VMOVDQA Y12, Y14 | |
| VMOVDQA Y4, Y12 | |
| VMOVDQA Y5, Y4 | |
| VMOVDQA Y9, Y5 | |
| VMOVDQA Y13, Y9 | |
| VMOVDQA Y1, Y13 | |
| VMOVDQA Y6, Y1 | |
| VMOVDQA Y10, Y6 | |
| JMP openAVX2ShortOpenLoop | |
| openAVX2ShortTail32: | |
| CMPQ BX, $0x10 | |
| VMOVDQA X0, X1 | |
| JB openAVX2ShortDone | |
| SUBQ $0x10, BX | |
| // Load for hashing | |
| ADDQ (SI), R10 | |
| ADCQ 8(SI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| // Load for decryption | |
| VPXOR (SI), X0, X12 | |
| VMOVDQU X12, (DI) | |
| LEAQ 16(SI), SI | |
| LEAQ 16(DI), DI | |
| VPERM2I128 $0x11, Y0, Y0, Y0 | |
| VMOVDQA X0, X1 | |
| openAVX2ShortDone: | |
| VZEROUPPER | |
| JMP openSSETail16 | |
| openAVX2320: | |
| VMOVDQA Y0, Y5 | |
| VMOVDQA Y14, Y9 | |
| VMOVDQA Y12, Y13 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 | |
| VMOVDQA Y0, Y6 | |
| VMOVDQA Y14, Y10 | |
| VMOVDQA Y12, Y8 | |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 | |
| VMOVDQA Y14, Y7 | |
| VMOVDQA Y12, Y11 | |
| VMOVDQA Y4, Y15 | |
| MOVQ $0x0000000a, R9 | |
| openAVX2320InnerCipherLoop: | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x0c, Y10, Y3 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y3, Y10, Y10 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x07, Y10, Y3 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y3, Y10, Y10 | |
| VPALIGNR $0x04, Y14, Y14, Y14 | |
| VPALIGNR $0x04, Y9, Y9, Y9 | |
| VPALIGNR $0x04, Y10, Y10, Y10 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x0c, Y4, Y4, Y4 | |
| VPALIGNR $0x0c, Y1, Y1, Y1 | |
| VPALIGNR $0x0c, Y2, Y2, Y2 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x0c, Y10, Y3 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y3, Y10, Y10 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x07, Y10, Y3 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y3, Y10, Y10 | |
| VPALIGNR $0x0c, Y14, Y14, Y14 | |
| VPALIGNR $0x0c, Y9, Y9, Y9 | |
| VPALIGNR $0x0c, Y10, Y10, Y10 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x04, Y4, Y4, Y4 | |
| VPALIGNR $0x04, Y1, Y1, Y1 | |
| VPALIGNR $0x04, Y2, Y2, Y2 | |
| DECQ R9 | |
| JNE openAVX2320InnerCipherLoop | |
| VMOVDQA ·chacha20Constants<>+0(SB), Y3 | |
| VPADDD Y3, Y0, Y0 | |
| VPADDD Y3, Y5, Y5 | |
| VPADDD Y3, Y6, Y6 | |
| VPADDD Y7, Y14, Y14 | |
| VPADDD Y7, Y9, Y9 | |
| VPADDD Y7, Y10, Y10 | |
| VPADDD Y11, Y12, Y12 | |
| VPADDD Y11, Y13, Y13 | |
| VPADDD Y11, Y8, Y8 | |
| VMOVDQA ·avx2IncMask<>+0(SB), Y3 | |
| VPADDD Y15, Y4, Y4 | |
| VPADDD Y3, Y15, Y15 | |
| VPADDD Y15, Y1, Y1 | |
| VPADDD Y3, Y15, Y15 | |
| VPADDD Y15, Y2, Y2 | |
| // Clamp and store poly key | |
| VPERM2I128 $0x02, Y0, Y14, Y3 | |
| VPAND ·polyClampMask<>+0(SB), Y3, Y3 | |
| VMOVDQA Y3, (BP) | |
| // Stream for up to 320 bytes | |
| VPERM2I128 $0x13, Y0, Y14, Y0 | |
| VPERM2I128 $0x13, Y12, Y4, Y14 | |
| VPERM2I128 $0x02, Y5, Y9, Y12 | |
| VPERM2I128 $0x02, Y13, Y1, Y4 | |
| VPERM2I128 $0x13, Y5, Y9, Y5 | |
| VPERM2I128 $0x13, Y13, Y1, Y9 | |
| VPERM2I128 $0x02, Y6, Y10, Y13 | |
| VPERM2I128 $0x02, Y8, Y2, Y1 | |
| VPERM2I128 $0x13, Y6, Y10, Y6 | |
| VPERM2I128 $0x13, Y8, Y2, Y10 | |
| JMP openAVX2ShortOpen | |
| openAVX2Tail128: | |
| // Need to decrypt up to 128 bytes - prepare two blocks | |
| VMOVDQA ·chacha20Constants<>+0(SB), Y5 | |
| VMOVDQA 32(BP), Y9 | |
| VMOVDQA 64(BP), Y13 | |
| VMOVDQA 192(BP), Y1 | |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y1 | |
| VMOVDQA Y1, Y4 | |
| XORQ R9, R9 | |
| MOVQ BX, CX | |
| ANDQ $-16, CX | |
| TESTQ CX, CX | |
| JE openAVX2Tail128LoopB | |
| openAVX2Tail128LoopA: | |
| ADDQ (SI)(R9*1), R10 | |
| ADCQ 8(SI)(R9*1), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| openAVX2Tail128LoopB: | |
| ADDQ $0x10, R9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPALIGNR $0x04, Y9, Y9, Y9 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x0c, Y1, Y1, Y1 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPALIGNR $0x0c, Y9, Y9, Y9 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x04, Y1, Y1, Y1 | |
| CMPQ R9, CX | |
| JB openAVX2Tail128LoopA | |
| CMPQ R9, $0xa0 | |
| JNE openAVX2Tail128LoopB | |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 | |
| VPADDD 32(BP), Y9, Y9 | |
| VPADDD 64(BP), Y13, Y13 | |
| VPADDD Y4, Y1, Y1 | |
| VPERM2I128 $0x02, Y5, Y9, Y0 | |
| VPERM2I128 $0x02, Y13, Y1, Y14 | |
| VPERM2I128 $0x13, Y5, Y9, Y12 | |
| VPERM2I128 $0x13, Y13, Y1, Y4 | |
| openAVX2TailLoop: | |
| CMPQ BX, $0x20 | |
| JB openAVX2Tail | |
| SUBQ $0x20, BX | |
| // Load for decryption | |
| VPXOR (SI), Y0, Y0 | |
| VMOVDQU Y0, (DI) | |
| LEAQ 32(SI), SI | |
| LEAQ 32(DI), DI | |
| VMOVDQA Y14, Y0 | |
| VMOVDQA Y12, Y14 | |
| VMOVDQA Y4, Y12 | |
| JMP openAVX2TailLoop | |
| openAVX2Tail: | |
| CMPQ BX, $0x10 | |
| VMOVDQA X0, X1 | |
| JB openAVX2TailDone | |
| SUBQ $0x10, BX | |
| // Load for decryption | |
| VPXOR (SI), X0, X12 | |
| VMOVDQU X12, (DI) | |
| LEAQ 16(SI), SI | |
| LEAQ 16(DI), DI | |
| VPERM2I128 $0x11, Y0, Y0, Y0 | |
| VMOVDQA X0, X1 | |
| openAVX2TailDone: | |
| VZEROUPPER | |
| JMP openSSETail16 | |
| openAVX2Tail256: | |
| VMOVDQA ·chacha20Constants<>+0(SB), Y0 | |
| VMOVDQA Y0, Y5 | |
| VMOVDQA 32(BP), Y14 | |
| VMOVDQA Y14, Y9 | |
| VMOVDQA 64(BP), Y12 | |
| VMOVDQA Y12, Y13 | |
| VMOVDQA 192(BP), Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 | |
| VMOVDQA Y4, Y7 | |
| VMOVDQA Y1, Y11 | |
| // Compute the number of iterations that will hash data | |
| MOVQ BX, 224(BP) | |
| MOVQ BX, CX | |
| SUBQ $0x80, CX | |
| SHRQ $0x04, CX | |
| MOVQ $0x0000000a, R9 | |
| CMPQ CX, $0x0a | |
| CMOVQGT R9, CX | |
| MOVQ SI, BX | |
| XORQ R9, R9 | |
| openAVX2Tail256LoopA: | |
| ADDQ (BX), R10 | |
| ADCQ 8(BX), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 16(BX), BX | |
| openAVX2Tail256LoopB: | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPALIGNR $0x04, Y14, Y14, Y14 | |
| VPALIGNR $0x04, Y9, Y9, Y9 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x0c, Y4, Y4, Y4 | |
| VPALIGNR $0x0c, Y1, Y1, Y1 | |
| INCQ R9 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPALIGNR $0x0c, Y14, Y14, Y14 | |
| VPALIGNR $0x0c, Y9, Y9, Y9 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x04, Y4, Y4, Y4 | |
| VPALIGNR $0x04, Y1, Y1, Y1 | |
| CMPQ R9, CX | |
| JB openAVX2Tail256LoopA | |
| CMPQ R9, $0x0a | |
| JNE openAVX2Tail256LoopB | |
| MOVQ BX, R9 | |
| SUBQ SI, BX | |
| MOVQ BX, CX | |
| MOVQ 224(BP), BX | |
| openAVX2Tail256Hash: | |
| ADDQ $0x10, CX | |
| CMPQ CX, BX | |
| JGT openAVX2Tail256HashEnd | |
| ADDQ (R9), R10 | |
| ADCQ 8(R9), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 16(R9), R9 | |
| JMP openAVX2Tail256Hash | |
| openAVX2Tail256HashEnd: | |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 | |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 | |
| VPADDD 32(BP), Y14, Y14 | |
| VPADDD 32(BP), Y9, Y9 | |
| VPADDD 64(BP), Y12, Y12 | |
| VPADDD 64(BP), Y13, Y13 | |
| VPADDD Y7, Y4, Y4 | |
| VPADDD Y11, Y1, Y1 | |
| VPERM2I128 $0x02, Y0, Y14, Y6 | |
| VPERM2I128 $0x02, Y12, Y4, Y10 | |
| VPERM2I128 $0x13, Y0, Y14, Y8 | |
| VPERM2I128 $0x13, Y12, Y4, Y2 | |
| VPERM2I128 $0x02, Y5, Y9, Y0 | |
| VPERM2I128 $0x02, Y13, Y1, Y14 | |
| VPERM2I128 $0x13, Y5, Y9, Y12 | |
| VPERM2I128 $0x13, Y13, Y1, Y4 | |
| VPXOR (SI), Y6, Y6 | |
| VPXOR 32(SI), Y10, Y10 | |
| VPXOR 64(SI), Y8, Y8 | |
| VPXOR 96(SI), Y2, Y2 | |
| VMOVDQU Y6, (DI) | |
| VMOVDQU Y10, 32(DI) | |
| VMOVDQU Y8, 64(DI) | |
| VMOVDQU Y2, 96(DI) | |
| LEAQ 128(SI), SI | |
| LEAQ 128(DI), DI | |
| SUBQ $0x80, BX | |
| JMP openAVX2TailLoop | |
| openAVX2Tail384: | |
| // Need to decrypt up to 384 bytes - prepare six blocks | |
| VMOVDQA ·chacha20Constants<>+0(SB), Y0 | |
| VMOVDQA Y0, Y5 | |
| VMOVDQA Y0, Y6 | |
| VMOVDQA 32(BP), Y14 | |
| VMOVDQA Y14, Y9 | |
| VMOVDQA Y14, Y10 | |
| VMOVDQA 64(BP), Y12 | |
| VMOVDQA Y12, Y13 | |
| VMOVDQA Y12, Y8 | |
| VMOVDQA 192(BP), Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 | |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 | |
| VMOVDQA Y4, 96(BP) | |
| VMOVDQA Y1, 128(BP) | |
| VMOVDQA Y2, 160(BP) | |
| // Compute the number of iterations that will hash two blocks of data | |
| MOVQ BX, 224(BP) | |
| MOVQ BX, CX | |
| SUBQ $0x00000100, CX | |
| SHRQ $0x04, CX | |
| ADDQ $0x06, CX | |
| MOVQ $0x0000000a, R9 | |
| CMPQ CX, $0x0a | |
| CMOVQGT R9, CX | |
| MOVQ SI, BX | |
| XORQ R9, R9 | |
| openAVX2Tail384LoopB: | |
| ADDQ (BX), R10 | |
| ADCQ 8(BX), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 16(BX), BX | |
| openAVX2Tail384LoopA: | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x0c, Y10, Y3 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y3, Y10, Y10 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x07, Y10, Y3 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y3, Y10, Y10 | |
| VPALIGNR $0x04, Y14, Y14, Y14 | |
| VPALIGNR $0x04, Y9, Y9, Y9 | |
| VPALIGNR $0x04, Y10, Y10, Y10 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x0c, Y4, Y4, Y4 | |
| VPALIGNR $0x0c, Y1, Y1, Y1 | |
| VPALIGNR $0x0c, Y2, Y2, Y2 | |
| ADDQ (BX), R10 | |
| ADCQ 8(BX), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 16(BX), BX | |
| INCQ R9 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x0c, Y10, Y3 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y3, Y10, Y10 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x07, Y10, Y3 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y3, Y10, Y10 | |
| VPALIGNR $0x0c, Y14, Y14, Y14 | |
| VPALIGNR $0x0c, Y9, Y9, Y9 | |
| VPALIGNR $0x0c, Y10, Y10, Y10 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x04, Y4, Y4, Y4 | |
| VPALIGNR $0x04, Y1, Y1, Y1 | |
| VPALIGNR $0x04, Y2, Y2, Y2 | |
| CMPQ R9, CX | |
| JB openAVX2Tail384LoopB | |
| CMPQ R9, $0x0a | |
| JNE openAVX2Tail384LoopA | |
| MOVQ BX, R9 | |
| SUBQ SI, BX | |
| MOVQ BX, CX | |
| MOVQ 224(BP), BX | |
| openAVX2Tail384Hash: | |
| ADDQ $0x10, CX | |
| CMPQ CX, BX | |
| JGT openAVX2Tail384HashEnd | |
| ADDQ (R9), R10 | |
| ADCQ 8(R9), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 16(R9), R9 | |
| JMP openAVX2Tail384Hash | |
| openAVX2Tail384HashEnd: | |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 | |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 | |
| VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 | |
| VPADDD 32(BP), Y14, Y14 | |
| VPADDD 32(BP), Y9, Y9 | |
| VPADDD 32(BP), Y10, Y10 | |
| VPADDD 64(BP), Y12, Y12 | |
| VPADDD 64(BP), Y13, Y13 | |
| VPADDD 64(BP), Y8, Y8 | |
| VPADDD 96(BP), Y4, Y4 | |
| VPADDD 128(BP), Y1, Y1 | |
| VPADDD 160(BP), Y2, Y2 | |
| VPERM2I128 $0x02, Y0, Y14, Y3 | |
| VPERM2I128 $0x02, Y12, Y4, Y7 | |
| VPERM2I128 $0x13, Y0, Y14, Y11 | |
| VPERM2I128 $0x13, Y12, Y4, Y15 | |
| VPXOR (SI), Y3, Y3 | |
| VPXOR 32(SI), Y7, Y7 | |
| VPXOR 64(SI), Y11, Y11 | |
| VPXOR 96(SI), Y15, Y15 | |
| VMOVDQU Y3, (DI) | |
| VMOVDQU Y7, 32(DI) | |
| VMOVDQU Y11, 64(DI) | |
| VMOVDQU Y15, 96(DI) | |
| VPERM2I128 $0x02, Y5, Y9, Y3 | |
| VPERM2I128 $0x02, Y13, Y1, Y7 | |
| VPERM2I128 $0x13, Y5, Y9, Y11 | |
| VPERM2I128 $0x13, Y13, Y1, Y15 | |
| VPXOR 128(SI), Y3, Y3 | |
| VPXOR 160(SI), Y7, Y7 | |
| VPXOR 192(SI), Y11, Y11 | |
| VPXOR 224(SI), Y15, Y15 | |
| VMOVDQU Y3, 128(DI) | |
| VMOVDQU Y7, 160(DI) | |
| VMOVDQU Y11, 192(DI) | |
| VMOVDQU Y15, 224(DI) | |
| VPERM2I128 $0x02, Y6, Y10, Y0 | |
| VPERM2I128 $0x02, Y8, Y2, Y14 | |
| VPERM2I128 $0x13, Y6, Y10, Y12 | |
| VPERM2I128 $0x13, Y8, Y2, Y4 | |
| LEAQ 256(SI), SI | |
| LEAQ 256(DI), DI | |
| SUBQ $0x00000100, BX | |
| JMP openAVX2TailLoop | |
| openAVX2Tail512: | |
| VMOVDQU ·chacha20Constants<>+0(SB), Y0 | |
| VMOVDQA Y0, Y5 | |
| VMOVDQA Y0, Y6 | |
| VMOVDQA Y0, Y7 | |
| VMOVDQA 32(BP), Y14 | |
| VMOVDQA Y14, Y9 | |
| VMOVDQA Y14, Y10 | |
| VMOVDQA Y14, Y11 | |
| VMOVDQA 64(BP), Y12 | |
| VMOVDQA Y12, Y13 | |
| VMOVDQA Y12, Y8 | |
| VMOVDQA Y12, Y15 | |
| VMOVDQA 192(BP), Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 | |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 | |
| VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 | |
| VMOVDQA Y4, 96(BP) | |
| VMOVDQA Y1, 128(BP) | |
| VMOVDQA Y2, 160(BP) | |
| VMOVDQA Y3, 192(BP) | |
| XORQ CX, CX | |
| MOVQ SI, R9 | |
| openAVX2Tail512LoopB: | |
| ADDQ (R9), R10 | |
| ADCQ 8(R9), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 16(R9), R9 | |
| openAVX2Tail512LoopA: | |
| VPADDD Y14, Y0, Y0 | |
| VPADDD Y9, Y5, Y5 | |
| VPADDD Y10, Y6, Y6 | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y0, Y4, Y4 | |
| VPXOR Y5, Y1, Y1 | |
| VPXOR Y6, Y2, Y2 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 | |
| VPADDD Y4, Y12, Y12 | |
| VPADDD Y1, Y13, Y13 | |
| VPADDD Y2, Y8, Y8 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y12, Y14, Y14 | |
| VPXOR Y13, Y9, Y9 | |
| VPXOR Y8, Y10, Y10 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA Y15, 224(BP) | |
| VPSLLD $0x0c, Y14, Y15 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPSLLD $0x0c, Y9, Y15 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPSLLD $0x0c, Y10, Y15 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPSLLD $0x0c, Y11, Y15 | |
| VPSRLD $0x14, Y11, Y11 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA 224(BP), Y15 | |
| ADDQ (R9), R10 | |
| ADCQ 8(R9), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| VPADDD Y14, Y0, Y0 | |
| VPADDD Y9, Y5, Y5 | |
| VPADDD Y10, Y6, Y6 | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y0, Y4, Y4 | |
| VPXOR Y5, Y1, Y1 | |
| VPXOR Y6, Y2, Y2 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 | |
| VPADDD Y4, Y12, Y12 | |
| VPADDD Y1, Y13, Y13 | |
| VPADDD Y2, Y8, Y8 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y12, Y14, Y14 | |
| VPXOR Y13, Y9, Y9 | |
| VPXOR Y8, Y10, Y10 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA Y15, 224(BP) | |
| VPSLLD $0x07, Y14, Y15 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPSLLD $0x07, Y9, Y15 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPSLLD $0x07, Y10, Y15 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPSLLD $0x07, Y11, Y15 | |
| VPSRLD $0x19, Y11, Y11 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA 224(BP), Y15 | |
| VPALIGNR $0x04, Y14, Y14, Y14 | |
| VPALIGNR $0x04, Y9, Y9, Y9 | |
| VPALIGNR $0x04, Y10, Y10, Y10 | |
| VPALIGNR $0x04, Y11, Y11, Y11 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x08, Y15, Y15, Y15 | |
| VPALIGNR $0x0c, Y4, Y4, Y4 | |
| VPALIGNR $0x0c, Y1, Y1, Y1 | |
| VPALIGNR $0x0c, Y2, Y2, Y2 | |
| VPALIGNR $0x0c, Y3, Y3, Y3 | |
| VPADDD Y14, Y0, Y0 | |
| VPADDD Y9, Y5, Y5 | |
| VPADDD Y10, Y6, Y6 | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y0, Y4, Y4 | |
| VPXOR Y5, Y1, Y1 | |
| VPXOR Y6, Y2, Y2 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 | |
| VPADDD Y4, Y12, Y12 | |
| VPADDD Y1, Y13, Y13 | |
| VPADDD Y2, Y8, Y8 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y12, Y14, Y14 | |
| VPXOR Y13, Y9, Y9 | |
| VPXOR Y8, Y10, Y10 | |
| VPXOR Y15, Y11, Y11 | |
| ADDQ 16(R9), R10 | |
| ADCQ 24(R9), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 32(R9), R9 | |
| VMOVDQA Y15, 224(BP) | |
| VPSLLD $0x0c, Y14, Y15 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPSLLD $0x0c, Y9, Y15 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPSLLD $0x0c, Y10, Y15 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPSLLD $0x0c, Y11, Y15 | |
| VPSRLD $0x14, Y11, Y11 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA 224(BP), Y15 | |
| VPADDD Y14, Y0, Y0 | |
| VPADDD Y9, Y5, Y5 | |
| VPADDD Y10, Y6, Y6 | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y0, Y4, Y4 | |
| VPXOR Y5, Y1, Y1 | |
| VPXOR Y6, Y2, Y2 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 | |
| VPADDD Y4, Y12, Y12 | |
| VPADDD Y1, Y13, Y13 | |
| VPADDD Y2, Y8, Y8 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y12, Y14, Y14 | |
| VPXOR Y13, Y9, Y9 | |
| VPXOR Y8, Y10, Y10 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA Y15, 224(BP) | |
| VPSLLD $0x07, Y14, Y15 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPSLLD $0x07, Y9, Y15 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPSLLD $0x07, Y10, Y15 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPSLLD $0x07, Y11, Y15 | |
| VPSRLD $0x19, Y11, Y11 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA 224(BP), Y15 | |
| VPALIGNR $0x0c, Y14, Y14, Y14 | |
| VPALIGNR $0x0c, Y9, Y9, Y9 | |
| VPALIGNR $0x0c, Y10, Y10, Y10 | |
| VPALIGNR $0x0c, Y11, Y11, Y11 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x08, Y15, Y15, Y15 | |
| VPALIGNR $0x04, Y4, Y4, Y4 | |
| VPALIGNR $0x04, Y1, Y1, Y1 | |
| VPALIGNR $0x04, Y2, Y2, Y2 | |
| VPALIGNR $0x04, Y3, Y3, Y3 | |
| INCQ CX | |
| CMPQ CX, $0x04 | |
| JLT openAVX2Tail512LoopB | |
| CMPQ CX, $0x0a | |
| JNE openAVX2Tail512LoopA | |
| MOVQ BX, CX | |
| SUBQ $0x00000180, CX | |
| ANDQ $-16, CX | |
| openAVX2Tail512HashLoop: | |
| TESTQ CX, CX | |
| JE openAVX2Tail512HashEnd | |
| ADDQ (R9), R10 | |
| ADCQ 8(R9), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 16(R9), R9 | |
| SUBQ $0x10, CX | |
| JMP openAVX2Tail512HashLoop | |
| openAVX2Tail512HashEnd: | |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 | |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 | |
| VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 | |
| VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 | |
| VPADDD 32(BP), Y14, Y14 | |
| VPADDD 32(BP), Y9, Y9 | |
| VPADDD 32(BP), Y10, Y10 | |
| VPADDD 32(BP), Y11, Y11 | |
| VPADDD 64(BP), Y12, Y12 | |
| VPADDD 64(BP), Y13, Y13 | |
| VPADDD 64(BP), Y8, Y8 | |
| VPADDD 64(BP), Y15, Y15 | |
| VPADDD 96(BP), Y4, Y4 | |
| VPADDD 128(BP), Y1, Y1 | |
| VPADDD 160(BP), Y2, Y2 | |
| VPADDD 192(BP), Y3, Y3 | |
| VMOVDQA Y15, 224(BP) | |
| VPERM2I128 $0x02, Y0, Y14, Y15 | |
| VPERM2I128 $0x13, Y0, Y14, Y14 | |
| VPERM2I128 $0x02, Y12, Y4, Y0 | |
| VPERM2I128 $0x13, Y12, Y4, Y12 | |
| VPXOR (SI), Y15, Y15 | |
| VPXOR 32(SI), Y0, Y0 | |
| VPXOR 64(SI), Y14, Y14 | |
| VPXOR 96(SI), Y12, Y12 | |
| VMOVDQU Y15, (DI) | |
| VMOVDQU Y0, 32(DI) | |
| VMOVDQU Y14, 64(DI) | |
| VMOVDQU Y12, 96(DI) | |
| VPERM2I128 $0x02, Y5, Y9, Y0 | |
| VPERM2I128 $0x02, Y13, Y1, Y14 | |
| VPERM2I128 $0x13, Y5, Y9, Y12 | |
| VPERM2I128 $0x13, Y13, Y1, Y4 | |
| VPXOR 128(SI), Y0, Y0 | |
| VPXOR 160(SI), Y14, Y14 | |
| VPXOR 192(SI), Y12, Y12 | |
| VPXOR 224(SI), Y4, Y4 | |
| VMOVDQU Y0, 128(DI) | |
| VMOVDQU Y14, 160(DI) | |
| VMOVDQU Y12, 192(DI) | |
| VMOVDQU Y4, 224(DI) | |
| VPERM2I128 $0x02, Y6, Y10, Y0 | |
| VPERM2I128 $0x02, Y8, Y2, Y14 | |
| VPERM2I128 $0x13, Y6, Y10, Y12 | |
| VPERM2I128 $0x13, Y8, Y2, Y4 | |
| VPXOR 256(SI), Y0, Y0 | |
| VPXOR 288(SI), Y14, Y14 | |
| VPXOR 320(SI), Y12, Y12 | |
| VPXOR 352(SI), Y4, Y4 | |
| VMOVDQU Y0, 256(DI) | |
| VMOVDQU Y14, 288(DI) | |
| VMOVDQU Y12, 320(DI) | |
| VMOVDQU Y4, 352(DI) | |
| VPERM2I128 $0x02, Y7, Y11, Y0 | |
| VPERM2I128 $0x02, 224(BP), Y3, Y14 | |
| VPERM2I128 $0x13, Y7, Y11, Y12 | |
| VPERM2I128 $0x13, 224(BP), Y3, Y4 | |
| LEAQ 384(SI), SI | |
| LEAQ 384(DI), DI | |
| SUBQ $0x00000180, BX | |
| JMP openAVX2TailLoop | |
| DATA ·chacha20Constants<>+0(SB)/4, $0x61707865 | |
| DATA ·chacha20Constants<>+4(SB)/4, $0x3320646e | |
| DATA ·chacha20Constants<>+8(SB)/4, $0x79622d32 | |
| DATA ·chacha20Constants<>+12(SB)/4, $0x6b206574 | |
| DATA ·chacha20Constants<>+16(SB)/4, $0x61707865 | |
| DATA ·chacha20Constants<>+20(SB)/4, $0x3320646e | |
| DATA ·chacha20Constants<>+24(SB)/4, $0x79622d32 | |
| DATA ·chacha20Constants<>+28(SB)/4, $0x6b206574 | |
| GLOBL ·chacha20Constants<>(SB), RODATA|NOPTR, $32 | |
| DATA ·avx2InitMask<>+0(SB)/8, $0x0000000000000000 | |
| DATA ·avx2InitMask<>+8(SB)/8, $0x0000000000000000 | |
| DATA ·avx2InitMask<>+16(SB)/8, $0x0000000000000001 | |
| DATA ·avx2InitMask<>+24(SB)/8, $0x0000000000000000 | |
| GLOBL ·avx2InitMask<>(SB), RODATA|NOPTR, $32 | |
| DATA ·rol16<>+0(SB)/8, $0x0504070601000302 | |
| DATA ·rol16<>+8(SB)/8, $0x0d0c0f0e09080b0a | |
| DATA ·rol16<>+16(SB)/8, $0x0504070601000302 | |
| DATA ·rol16<>+24(SB)/8, $0x0d0c0f0e09080b0a | |
| GLOBL ·rol16<>(SB), RODATA|NOPTR, $32 | |
| DATA ·rol8<>+0(SB)/8, $0x0605040702010003 | |
| DATA ·rol8<>+8(SB)/8, $0x0e0d0c0f0a09080b | |
| DATA ·rol8<>+16(SB)/8, $0x0605040702010003 | |
| DATA ·rol8<>+24(SB)/8, $0x0e0d0c0f0a09080b | |
| GLOBL ·rol8<>(SB), RODATA|NOPTR, $32 | |
| DATA ·polyClampMask<>+0(SB)/8, $0x0ffffffc0fffffff | |
| DATA ·polyClampMask<>+8(SB)/8, $0x0ffffffc0ffffffc | |
| DATA ·polyClampMask<>+16(SB)/8, $0xffffffffffffffff | |
| DATA ·polyClampMask<>+24(SB)/8, $0xffffffffffffffff | |
| GLOBL ·polyClampMask<>(SB), RODATA|NOPTR, $32 | |
| DATA ·avx2IncMask<>+0(SB)/8, $0x0000000000000002 | |
| DATA ·avx2IncMask<>+8(SB)/8, $0x0000000000000000 | |
| DATA ·avx2IncMask<>+16(SB)/8, $0x0000000000000002 | |
| DATA ·avx2IncMask<>+24(SB)/8, $0x0000000000000000 | |
| GLOBL ·avx2IncMask<>(SB), RODATA|NOPTR, $32 | |
| DATA ·andMask<>+0(SB)/8, $0x00000000000000ff | |
| DATA ·andMask<>+8(SB)/8, $0x0000000000000000 | |
| DATA ·andMask<>+16(SB)/8, $0x000000000000ffff | |
| DATA ·andMask<>+24(SB)/8, $0x0000000000000000 | |
| DATA ·andMask<>+32(SB)/8, $0x0000000000ffffff | |
| DATA ·andMask<>+40(SB)/8, $0x0000000000000000 | |
| DATA ·andMask<>+48(SB)/8, $0x00000000ffffffff | |
| DATA ·andMask<>+56(SB)/8, $0x0000000000000000 | |
| DATA ·andMask<>+64(SB)/8, $0x000000ffffffffff | |
| DATA ·andMask<>+72(SB)/8, $0x0000000000000000 | |
| DATA ·andMask<>+80(SB)/8, $0x0000ffffffffffff | |
| DATA ·andMask<>+88(SB)/8, $0x0000000000000000 | |
| DATA ·andMask<>+96(SB)/8, $0x00ffffffffffffff | |
| DATA ·andMask<>+104(SB)/8, $0x0000000000000000 | |
| DATA ·andMask<>+112(SB)/8, $0xffffffffffffffff | |
| DATA ·andMask<>+120(SB)/8, $0x0000000000000000 | |
| DATA ·andMask<>+128(SB)/8, $0xffffffffffffffff | |
| DATA ·andMask<>+136(SB)/8, $0x00000000000000ff | |
| DATA ·andMask<>+144(SB)/8, $0xffffffffffffffff | |
| DATA ·andMask<>+152(SB)/8, $0x000000000000ffff | |
| DATA ·andMask<>+160(SB)/8, $0xffffffffffffffff | |
| DATA ·andMask<>+168(SB)/8, $0x0000000000ffffff | |
| DATA ·andMask<>+176(SB)/8, $0xffffffffffffffff | |
| DATA ·andMask<>+184(SB)/8, $0x00000000ffffffff | |
| DATA ·andMask<>+192(SB)/8, $0xffffffffffffffff | |
| DATA ·andMask<>+200(SB)/8, $0x000000ffffffffff | |
| DATA ·andMask<>+208(SB)/8, $0xffffffffffffffff | |
| DATA ·andMask<>+216(SB)/8, $0x0000ffffffffffff | |
| DATA ·andMask<>+224(SB)/8, $0xffffffffffffffff | |
| DATA ·andMask<>+232(SB)/8, $0x00ffffffffffffff | |
| GLOBL ·andMask<>(SB), RODATA|NOPTR, $240 | |
| // func chacha20Poly1305Seal(dst []byte, key []uint32, src []byte, ad []byte) | |
| // Requires: AVX, AVX2, BMI2, CMOV, SSE2 | |
| TEXT ·chacha20Poly1305Seal(SB), $288-96 | |
| MOVQ SP, BP | |
| ADDQ $0x20, BP | |
| ANDQ $-32, BP | |
| MOVQ dst_base+0(FP), DI | |
| MOVQ key_base+24(FP), R8 | |
| MOVQ src_base+48(FP), SI | |
| MOVQ src_len+56(FP), BX | |
| MOVQ ad_base+72(FP), CX | |
| VZEROUPPER | |
| VMOVDQU ·chacha20Constants<>+0(SB), Y0 | |
| VBROADCASTI128 16(R8), Y14 | |
| VBROADCASTI128 32(R8), Y12 | |
| VBROADCASTI128 48(R8), Y4 | |
| VPADDD ·avx2InitMask<>+0(SB), Y4, Y4 | |
| // Special optimizations, for very short buffers | |
| CMPQ BX, $0x000000c0 | |
| JBE seal192AVX2 | |
| CMPQ BX, $0x00000140 | |
| JBE seal320AVX2 | |
| // For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream | |
| VMOVDQA Y0, Y5 | |
| VMOVDQA Y0, Y6 | |
| VMOVDQA Y0, Y7 | |
| VMOVDQA Y14, Y9 | |
| VMOVDQA Y14, Y10 | |
| VMOVDQA Y14, Y11 | |
| VMOVDQA Y14, 32(BP) | |
| VMOVDQA Y12, Y13 | |
| VMOVDQA Y12, Y8 | |
| VMOVDQA Y12, Y15 | |
| VMOVDQA Y12, 64(BP) | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 | |
| VMOVDQA Y4, 96(BP) | |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 | |
| VMOVDQA Y1, 128(BP) | |
| VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 | |
| VMOVDQA Y2, 160(BP) | |
| VMOVDQA Y3, 192(BP) | |
| MOVQ $0x0000000a, R9 | |
| sealAVX2IntroLoop: | |
| VMOVDQA Y15, 224(BP) | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y15 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y15 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y15 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y15 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x0c, Y10, Y15 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x07, Y10, Y15 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VMOVDQA 224(BP), Y15 | |
| VMOVDQA Y13, 224(BP) | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y15, Y11, Y11 | |
| VPSLLD $0x0c, Y11, Y13 | |
| VPSRLD $0x14, Y11, Y11 | |
| VPXOR Y13, Y11, Y11 | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y15, Y11, Y11 | |
| VPSLLD $0x07, Y11, Y13 | |
| VPSRLD $0x19, Y11, Y11 | |
| VPXOR Y13, Y11, Y11 | |
| VMOVDQA 224(BP), Y13 | |
| VPALIGNR $0x04, Y14, Y14, Y14 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x0c, Y4, Y4, Y4 | |
| VPALIGNR $0x04, Y9, Y9, Y9 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x0c, Y1, Y1, Y1 | |
| VPALIGNR $0x04, Y10, Y10, Y10 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x0c, Y2, Y2, Y2 | |
| VPALIGNR $0x04, Y11, Y11, Y11 | |
| VPALIGNR $0x08, Y15, Y15, Y15 | |
| VPALIGNR $0x0c, Y3, Y3, Y3 | |
| VMOVDQA Y15, 224(BP) | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y15 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y15 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y15 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y15 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x0c, Y10, Y15 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x07, Y10, Y15 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VMOVDQA 224(BP), Y15 | |
| VMOVDQA Y13, 224(BP) | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y15, Y11, Y11 | |
| VPSLLD $0x0c, Y11, Y13 | |
| VPSRLD $0x14, Y11, Y11 | |
| VPXOR Y13, Y11, Y11 | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y15, Y11, Y11 | |
| VPSLLD $0x07, Y11, Y13 | |
| VPSRLD $0x19, Y11, Y11 | |
| VPXOR Y13, Y11, Y11 | |
| VMOVDQA 224(BP), Y13 | |
| VPALIGNR $0x0c, Y14, Y14, Y14 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x04, Y4, Y4, Y4 | |
| VPALIGNR $0x0c, Y9, Y9, Y9 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x04, Y1, Y1, Y1 | |
| VPALIGNR $0x0c, Y10, Y10, Y10 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x04, Y2, Y2, Y2 | |
| VPALIGNR $0x0c, Y11, Y11, Y11 | |
| VPALIGNR $0x08, Y15, Y15, Y15 | |
| VPALIGNR $0x04, Y3, Y3, Y3 | |
| DECQ R9 | |
| JNE sealAVX2IntroLoop | |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 | |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 | |
| VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 | |
| VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 | |
| VPADDD 32(BP), Y14, Y14 | |
| VPADDD 32(BP), Y9, Y9 | |
| VPADDD 32(BP), Y10, Y10 | |
| VPADDD 32(BP), Y11, Y11 | |
| VPADDD 64(BP), Y12, Y12 | |
| VPADDD 64(BP), Y13, Y13 | |
| VPADDD 64(BP), Y8, Y8 | |
| VPADDD 64(BP), Y15, Y15 | |
| VPADDD 96(BP), Y4, Y4 | |
| VPADDD 128(BP), Y1, Y1 | |
| VPADDD 160(BP), Y2, Y2 | |
| VPADDD 192(BP), Y3, Y3 | |
| VPERM2I128 $0x13, Y12, Y4, Y12 | |
| VPERM2I128 $0x02, Y0, Y14, Y4 | |
| VPERM2I128 $0x13, Y0, Y14, Y0 | |
| // Clamp and store poly key | |
| VPAND ·polyClampMask<>+0(SB), Y4, Y4 | |
| VMOVDQA Y4, (BP) | |
| // Hash AD | |
| MOVQ ad_len+80(FP), R9 | |
| CALL polyHashADInternal<>(SB) | |
| // Can store at least 320 bytes | |
| VPXOR (SI), Y0, Y0 | |
| VPXOR 32(SI), Y12, Y12 | |
| VMOVDQU Y0, (DI) | |
| VMOVDQU Y12, 32(DI) | |
| VPERM2I128 $0x02, Y5, Y9, Y0 | |
| VPERM2I128 $0x02, Y13, Y1, Y14 | |
| VPERM2I128 $0x13, Y5, Y9, Y12 | |
| VPERM2I128 $0x13, Y13, Y1, Y4 | |
| VPXOR 64(SI), Y0, Y0 | |
| VPXOR 96(SI), Y14, Y14 | |
| VPXOR 128(SI), Y12, Y12 | |
| VPXOR 160(SI), Y4, Y4 | |
| VMOVDQU Y0, 64(DI) | |
| VMOVDQU Y14, 96(DI) | |
| VMOVDQU Y12, 128(DI) | |
| VMOVDQU Y4, 160(DI) | |
| VPERM2I128 $0x02, Y6, Y10, Y0 | |
| VPERM2I128 $0x02, Y8, Y2, Y14 | |
| VPERM2I128 $0x13, Y6, Y10, Y12 | |
| VPERM2I128 $0x13, Y8, Y2, Y4 | |
| VPXOR 192(SI), Y0, Y0 | |
| VPXOR 224(SI), Y14, Y14 | |
| VPXOR 256(SI), Y12, Y12 | |
| VPXOR 288(SI), Y4, Y4 | |
| VMOVDQU Y0, 192(DI) | |
| VMOVDQU Y14, 224(DI) | |
| VMOVDQU Y12, 256(DI) | |
| VMOVDQU Y4, 288(DI) | |
| MOVQ $0x00000140, CX | |
| SUBQ $0x00000140, BX | |
| LEAQ 320(SI), SI | |
| VPERM2I128 $0x02, Y7, Y11, Y0 | |
| VPERM2I128 $0x02, Y15, Y3, Y14 | |
| VPERM2I128 $0x13, Y7, Y11, Y12 | |
| VPERM2I128 $0x13, Y15, Y3, Y4 | |
| CMPQ BX, $0x80 | |
| JBE sealAVX2SealHash | |
| VPXOR (SI), Y0, Y0 | |
| VPXOR 32(SI), Y14, Y14 | |
| VPXOR 64(SI), Y12, Y12 | |
| VPXOR 96(SI), Y4, Y4 | |
| VMOVDQU Y0, 320(DI) | |
| VMOVDQU Y14, 352(DI) | |
| VMOVDQU Y12, 384(DI) | |
| VMOVDQU Y4, 416(DI) | |
| SUBQ $0x80, BX | |
| LEAQ 128(SI), SI | |
| MOVQ $0x00000008, CX | |
| MOVQ $0x00000002, R9 | |
| CMPQ BX, $0x80 | |
| JBE sealAVX2Tail128 | |
| CMPQ BX, $0x00000100 | |
| JBE sealAVX2Tail256 | |
| CMPQ BX, $0x00000180 | |
| JBE sealAVX2Tail384 | |
| CMPQ BX, $0x00000200 | |
| JBE sealAVX2Tail512 | |
| // We have 448 bytes to hash, but main loop hashes 512 bytes at a time - perform some rounds, before the main loop | |
| VMOVDQA ·chacha20Constants<>+0(SB), Y0 | |
| VMOVDQA Y0, Y5 | |
| VMOVDQA Y0, Y6 | |
| VMOVDQA Y0, Y7 | |
| VMOVDQA 32(BP), Y14 | |
| VMOVDQA Y14, Y9 | |
| VMOVDQA Y14, Y10 | |
| VMOVDQA Y14, Y11 | |
| VMOVDQA 64(BP), Y12 | |
| VMOVDQA Y12, Y13 | |
| VMOVDQA Y12, Y8 | |
| VMOVDQA Y12, Y15 | |
| VMOVDQA 192(BP), Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 | |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 | |
| VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 | |
| VMOVDQA Y4, 96(BP) | |
| VMOVDQA Y1, 128(BP) | |
| VMOVDQA Y2, 160(BP) | |
| VMOVDQA Y3, 192(BP) | |
| VMOVDQA Y15, 224(BP) | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y15 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y15 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y15 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y15 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x0c, Y10, Y15 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x07, Y10, Y15 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VMOVDQA 224(BP), Y15 | |
| VMOVDQA Y13, 224(BP) | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y15, Y11, Y11 | |
| VPSLLD $0x0c, Y11, Y13 | |
| VPSRLD $0x14, Y11, Y11 | |
| VPXOR Y13, Y11, Y11 | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y15, Y11, Y11 | |
| VPSLLD $0x07, Y11, Y13 | |
| VPSRLD $0x19, Y11, Y11 | |
| VPXOR Y13, Y11, Y11 | |
| VMOVDQA 224(BP), Y13 | |
| VPALIGNR $0x04, Y14, Y14, Y14 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x0c, Y4, Y4, Y4 | |
| VPALIGNR $0x04, Y9, Y9, Y9 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x0c, Y1, Y1, Y1 | |
| VPALIGNR $0x04, Y10, Y10, Y10 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x0c, Y2, Y2, Y2 | |
| VPALIGNR $0x04, Y11, Y11, Y11 | |
| VPALIGNR $0x08, Y15, Y15, Y15 | |
| VPALIGNR $0x0c, Y3, Y3, Y3 | |
| VMOVDQA Y15, 224(BP) | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y15 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y15 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y15 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y15 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x0c, Y10, Y15 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x07, Y10, Y15 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VMOVDQA 224(BP), Y15 | |
| VMOVDQA Y13, 224(BP) | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y15, Y11, Y11 | |
| VPSLLD $0x0c, Y11, Y13 | |
| VPSRLD $0x14, Y11, Y11 | |
| VPXOR Y13, Y11, Y11 | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y15, Y11, Y11 | |
| VPSLLD $0x07, Y11, Y13 | |
| VPSRLD $0x19, Y11, Y11 | |
| VPXOR Y13, Y11, Y11 | |
| VMOVDQA 224(BP), Y13 | |
| VPALIGNR $0x0c, Y14, Y14, Y14 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x04, Y4, Y4, Y4 | |
| VPALIGNR $0x0c, Y9, Y9, Y9 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x04, Y1, Y1, Y1 | |
| VPALIGNR $0x0c, Y10, Y10, Y10 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x04, Y2, Y2, Y2 | |
| VPALIGNR $0x0c, Y11, Y11, Y11 | |
| VPALIGNR $0x08, Y15, Y15, Y15 | |
| VPALIGNR $0x04, Y3, Y3, Y3 | |
| VPADDD Y14, Y0, Y0 | |
| VPADDD Y9, Y5, Y5 | |
| VPADDD Y10, Y6, Y6 | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y0, Y4, Y4 | |
| VPXOR Y5, Y1, Y1 | |
| VPXOR Y6, Y2, Y2 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 | |
| VPADDD Y4, Y12, Y12 | |
| VPADDD Y1, Y13, Y13 | |
| VPADDD Y2, Y8, Y8 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y12, Y14, Y14 | |
| VPXOR Y13, Y9, Y9 | |
| VPXOR Y8, Y10, Y10 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA Y15, 224(BP) | |
| VPSLLD $0x0c, Y14, Y15 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPSLLD $0x0c, Y9, Y15 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPSLLD $0x0c, Y10, Y15 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPSLLD $0x0c, Y11, Y15 | |
| VPSRLD $0x14, Y11, Y11 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA 224(BP), Y15 | |
| SUBQ $0x10, DI | |
| MOVQ $0x00000009, CX | |
| JMP sealAVX2InternalLoopStart | |
| sealAVX2MainLoop: | |
| VMOVDQU ·chacha20Constants<>+0(SB), Y0 | |
| VMOVDQA Y0, Y5 | |
| VMOVDQA Y0, Y6 | |
| VMOVDQA Y0, Y7 | |
| VMOVDQA 32(BP), Y14 | |
| VMOVDQA Y14, Y9 | |
| VMOVDQA Y14, Y10 | |
| VMOVDQA Y14, Y11 | |
| VMOVDQA 64(BP), Y12 | |
| VMOVDQA Y12, Y13 | |
| VMOVDQA Y12, Y8 | |
| VMOVDQA Y12, Y15 | |
| VMOVDQA 192(BP), Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 | |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 | |
| VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 | |
| VMOVDQA Y4, 96(BP) | |
| VMOVDQA Y1, 128(BP) | |
| VMOVDQA Y2, 160(BP) | |
| VMOVDQA Y3, 192(BP) | |
| MOVQ $0x0000000a, CX | |
| sealAVX2InternalLoop: | |
| ADDQ (DI), R10 | |
| ADCQ 8(DI), R11 | |
| ADCQ $0x01, R12 | |
| VPADDD Y14, Y0, Y0 | |
| VPADDD Y9, Y5, Y5 | |
| VPADDD Y10, Y6, Y6 | |
| VPADDD Y11, Y7, Y7 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| VPXOR Y0, Y4, Y4 | |
| VPXOR Y5, Y1, Y1 | |
| VPXOR Y6, Y2, Y2 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| VPADDD Y4, Y12, Y12 | |
| VPADDD Y1, Y13, Y13 | |
| VPADDD Y2, Y8, Y8 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y12, Y14, Y14 | |
| VPXOR Y13, Y9, Y9 | |
| VPXOR Y8, Y10, Y10 | |
| VPXOR Y15, Y11, Y11 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| VMOVDQA Y15, 224(BP) | |
| VPSLLD $0x0c, Y14, Y15 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPSLLD $0x0c, Y9, Y15 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPSLLD $0x0c, Y10, Y15 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPSLLD $0x0c, Y11, Y15 | |
| VPSRLD $0x14, Y11, Y11 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA 224(BP), Y15 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| sealAVX2InternalLoopStart: | |
| VPADDD Y14, Y0, Y0 | |
| VPADDD Y9, Y5, Y5 | |
| VPADDD Y10, Y6, Y6 | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y0, Y4, Y4 | |
| VPXOR Y5, Y1, Y1 | |
| VPXOR Y6, Y2, Y2 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 | |
| ADDQ 16(DI), R10 | |
| ADCQ 24(DI), R11 | |
| ADCQ $0x01, R12 | |
| VPADDD Y4, Y12, Y12 | |
| VPADDD Y1, Y13, Y13 | |
| VPADDD Y2, Y8, Y8 | |
| VPADDD Y3, Y15, Y15 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| VPXOR Y12, Y14, Y14 | |
| VPXOR Y13, Y9, Y9 | |
| VPXOR Y8, Y10, Y10 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA Y15, 224(BP) | |
| VPSLLD $0x07, Y14, Y15 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPSLLD $0x07, Y9, Y15 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPSLLD $0x07, Y10, Y15 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPSLLD $0x07, Y11, Y15 | |
| VPSRLD $0x19, Y11, Y11 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA 224(BP), Y15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| VPALIGNR $0x04, Y14, Y14, Y14 | |
| VPALIGNR $0x04, Y9, Y9, Y9 | |
| VPALIGNR $0x04, Y10, Y10, Y10 | |
| VPALIGNR $0x04, Y11, Y11, Y11 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x08, Y15, Y15, Y15 | |
| VPALIGNR $0x0c, Y4, Y4, Y4 | |
| VPALIGNR $0x0c, Y1, Y1, Y1 | |
| VPALIGNR $0x0c, Y2, Y2, Y2 | |
| VPALIGNR $0x0c, Y3, Y3, Y3 | |
| VPADDD Y14, Y0, Y0 | |
| VPADDD Y9, Y5, Y5 | |
| VPADDD Y10, Y6, Y6 | |
| VPADDD Y11, Y7, Y7 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| VPXOR Y0, Y4, Y4 | |
| VPXOR Y5, Y1, Y1 | |
| VPXOR Y6, Y2, Y2 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| VPADDD Y4, Y12, Y12 | |
| VPADDD Y1, Y13, Y13 | |
| VPADDD Y2, Y8, Y8 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y12, Y14, Y14 | |
| VPXOR Y13, Y9, Y9 | |
| VPXOR Y8, Y10, Y10 | |
| VPXOR Y15, Y11, Y11 | |
| ADDQ 32(DI), R10 | |
| ADCQ 40(DI), R11 | |
| ADCQ $0x01, R12 | |
| LEAQ 48(DI), DI | |
| VMOVDQA Y15, 224(BP) | |
| VPSLLD $0x0c, Y14, Y15 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPSLLD $0x0c, Y9, Y15 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPSLLD $0x0c, Y10, Y15 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPSLLD $0x0c, Y11, Y15 | |
| VPSRLD $0x14, Y11, Y11 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA 224(BP), Y15 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| VPADDD Y14, Y0, Y0 | |
| VPADDD Y9, Y5, Y5 | |
| VPADDD Y10, Y6, Y6 | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y0, Y4, Y4 | |
| VPXOR Y5, Y1, Y1 | |
| VPXOR Y6, Y2, Y2 | |
| VPXOR Y7, Y3, Y3 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 | |
| VPADDD Y4, Y12, Y12 | |
| VPADDD Y1, Y13, Y13 | |
| VPADDD Y2, Y8, Y8 | |
| VPADDD Y3, Y15, Y15 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| VPXOR Y12, Y14, Y14 | |
| VPXOR Y13, Y9, Y9 | |
| VPXOR Y8, Y10, Y10 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA Y15, 224(BP) | |
| VPSLLD $0x07, Y14, Y15 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPSLLD $0x07, Y9, Y15 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPSLLD $0x07, Y10, Y15 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPSLLD $0x07, Y11, Y15 | |
| VPSRLD $0x19, Y11, Y11 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA 224(BP), Y15 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| VPALIGNR $0x0c, Y14, Y14, Y14 | |
| VPALIGNR $0x0c, Y9, Y9, Y9 | |
| VPALIGNR $0x0c, Y10, Y10, Y10 | |
| VPALIGNR $0x0c, Y11, Y11, Y11 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x08, Y15, Y15, Y15 | |
| VPALIGNR $0x04, Y4, Y4, Y4 | |
| VPALIGNR $0x04, Y1, Y1, Y1 | |
| VPALIGNR $0x04, Y2, Y2, Y2 | |
| VPALIGNR $0x04, Y3, Y3, Y3 | |
| DECQ CX | |
| JNE sealAVX2InternalLoop | |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 | |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 | |
| VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 | |
| VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 | |
| VPADDD 32(BP), Y14, Y14 | |
| VPADDD 32(BP), Y9, Y9 | |
| VPADDD 32(BP), Y10, Y10 | |
| VPADDD 32(BP), Y11, Y11 | |
| VPADDD 64(BP), Y12, Y12 | |
| VPADDD 64(BP), Y13, Y13 | |
| VPADDD 64(BP), Y8, Y8 | |
| VPADDD 64(BP), Y15, Y15 | |
| VPADDD 96(BP), Y4, Y4 | |
| VPADDD 128(BP), Y1, Y1 | |
| VPADDD 160(BP), Y2, Y2 | |
| VPADDD 192(BP), Y3, Y3 | |
| VMOVDQA Y15, 224(BP) | |
| // We only hashed 480 of the 512 bytes available - hash the remaining 32 here | |
| ADDQ (DI), R10 | |
| ADCQ 8(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 32(DI), DI | |
| VPERM2I128 $0x02, Y0, Y14, Y15 | |
| VPERM2I128 $0x13, Y0, Y14, Y14 | |
| VPERM2I128 $0x02, Y12, Y4, Y0 | |
| VPERM2I128 $0x13, Y12, Y4, Y12 | |
| VPXOR (SI), Y15, Y15 | |
| VPXOR 32(SI), Y0, Y0 | |
| VPXOR 64(SI), Y14, Y14 | |
| VPXOR 96(SI), Y12, Y12 | |
| VMOVDQU Y15, (DI) | |
| VMOVDQU Y0, 32(DI) | |
| VMOVDQU Y14, 64(DI) | |
| VMOVDQU Y12, 96(DI) | |
| VPERM2I128 $0x02, Y5, Y9, Y0 | |
| VPERM2I128 $0x02, Y13, Y1, Y14 | |
| VPERM2I128 $0x13, Y5, Y9, Y12 | |
| VPERM2I128 $0x13, Y13, Y1, Y4 | |
| VPXOR 128(SI), Y0, Y0 | |
| VPXOR 160(SI), Y14, Y14 | |
| VPXOR 192(SI), Y12, Y12 | |
| VPXOR 224(SI), Y4, Y4 | |
| VMOVDQU Y0, 128(DI) | |
| VMOVDQU Y14, 160(DI) | |
| VMOVDQU Y12, 192(DI) | |
| VMOVDQU Y4, 224(DI) | |
| // and here | |
| ADDQ -16(DI), R10 | |
| ADCQ -8(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| VPERM2I128 $0x02, Y6, Y10, Y0 | |
| VPERM2I128 $0x02, Y8, Y2, Y14 | |
| VPERM2I128 $0x13, Y6, Y10, Y12 | |
| VPERM2I128 $0x13, Y8, Y2, Y4 | |
| VPXOR 256(SI), Y0, Y0 | |
| VPXOR 288(SI), Y14, Y14 | |
| VPXOR 320(SI), Y12, Y12 | |
| VPXOR 352(SI), Y4, Y4 | |
| VMOVDQU Y0, 256(DI) | |
| VMOVDQU Y14, 288(DI) | |
| VMOVDQU Y12, 320(DI) | |
| VMOVDQU Y4, 352(DI) | |
| VPERM2I128 $0x02, Y7, Y11, Y0 | |
| VPERM2I128 $0x02, 224(BP), Y3, Y14 | |
| VPERM2I128 $0x13, Y7, Y11, Y12 | |
| VPERM2I128 $0x13, 224(BP), Y3, Y4 | |
| VPXOR 384(SI), Y0, Y0 | |
| VPXOR 416(SI), Y14, Y14 | |
| VPXOR 448(SI), Y12, Y12 | |
| VPXOR 480(SI), Y4, Y4 | |
| VMOVDQU Y0, 384(DI) | |
| VMOVDQU Y14, 416(DI) | |
| VMOVDQU Y12, 448(DI) | |
| VMOVDQU Y4, 480(DI) | |
| LEAQ 512(SI), SI | |
| SUBQ $0x00000200, BX | |
| CMPQ BX, $0x00000200 | |
| JG sealAVX2MainLoop | |
| // Tail can only hash 480 bytes | |
| ADDQ (DI), R10 | |
| ADCQ 8(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ 16(DI), R10 | |
| ADCQ 24(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 32(DI), DI | |
| MOVQ $0x0000000a, CX | |
| MOVQ $0x00000000, R9 | |
| CMPQ BX, $0x80 | |
| JBE sealAVX2Tail128 | |
| CMPQ BX, $0x00000100 | |
| JBE sealAVX2Tail256 | |
| CMPQ BX, $0x00000180 | |
| JBE sealAVX2Tail384 | |
| JMP sealAVX2Tail512 | |
| sealSSETail: | |
| TESTQ BX, BX | |
| JE sealSSEFinalize | |
| // We can only load the PT one byte at a time to avoid read after end of buffer | |
| MOVQ BX, R9 | |
| SHLQ $0x04, R9 | |
| LEAQ ·andMask<>+0(SB), R13 | |
| MOVQ BX, CX | |
| LEAQ -1(SI)(BX*1), SI | |
| XORQ R15, R15 | |
| XORQ R8, R8 | |
| XORQ AX, AX | |
| sealSSETailLoadLoop: | |
| SHLQ $0x08, R15, R8 | |
| SHLQ $0x08, R15 | |
| MOVB (SI), AX | |
| XORQ AX, R15 | |
| LEAQ -1(SI), SI | |
| DECQ CX | |
| JNE sealSSETailLoadLoop | |
| MOVQ R15, 64(BP) | |
| MOVQ R8, 72(BP) | |
| PXOR 64(BP), X1 | |
| MOVOU X1, (DI) | |
| MOVOU -16(R13)(R9*1), X12 | |
| PAND X12, X1 | |
| MOVQ X1, R13 | |
| PSRLDQ $0x08, X1 | |
| MOVQ X1, R14 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ BX, DI | |
| sealSSEFinalize: | |
| // Hash in the buffer lengths | |
| ADDQ ad_len+80(FP), R10 | |
| ADCQ src_len+56(FP), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| // Final reduce | |
| MOVQ R10, R13 | |
| MOVQ R11, R14 | |
| MOVQ R12, R15 | |
| SUBQ $-5, R10 | |
| SBBQ $-1, R11 | |
| SBBQ $0x03, R12 | |
| CMOVQCS R13, R10 | |
| CMOVQCS R14, R11 | |
| CMOVQCS R15, R12 | |
| // Add in the "s" part of the key | |
| ADDQ 16(BP), R10 | |
| ADCQ 24(BP), R11 | |
| // Finally store the tag at the end of the message | |
| MOVQ R10, (DI) | |
| MOVQ R11, 8(DI) | |
| RET | |
| seal192AVX2: | |
| VMOVDQA Y0, Y5 | |
| VMOVDQA Y14, Y9 | |
| VMOVDQA Y12, Y13 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 | |
| VMOVDQA Y0, Y6 | |
| VMOVDQA Y14, Y10 | |
| VMOVDQA Y12, Y8 | |
| VMOVDQA Y4, Y2 | |
| VMOVDQA Y1, Y15 | |
| MOVQ $0x0000000a, R9 | |
| sealAVX2192InnerCipherLoop: | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPALIGNR $0x04, Y14, Y14, Y14 | |
| VPALIGNR $0x04, Y9, Y9, Y9 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x0c, Y4, Y4, Y4 | |
| VPALIGNR $0x0c, Y1, Y1, Y1 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPALIGNR $0x0c, Y14, Y14, Y14 | |
| VPALIGNR $0x0c, Y9, Y9, Y9 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x04, Y4, Y4, Y4 | |
| VPALIGNR $0x04, Y1, Y1, Y1 | |
| DECQ R9 | |
| JNE sealAVX2192InnerCipherLoop | |
| VPADDD Y6, Y0, Y0 | |
| VPADDD Y6, Y5, Y5 | |
| VPADDD Y10, Y14, Y14 | |
| VPADDD Y10, Y9, Y9 | |
| VPADDD Y8, Y12, Y12 | |
| VPADDD Y8, Y13, Y13 | |
| VPADDD Y2, Y4, Y4 | |
| VPADDD Y15, Y1, Y1 | |
| VPERM2I128 $0x02, Y0, Y14, Y3 | |
| // Clamp and store poly key | |
| VPAND ·polyClampMask<>+0(SB), Y3, Y3 | |
| VMOVDQA Y3, (BP) | |
| // Stream for up to 192 bytes | |
| VPERM2I128 $0x13, Y0, Y14, Y0 | |
| VPERM2I128 $0x13, Y12, Y4, Y14 | |
| VPERM2I128 $0x02, Y5, Y9, Y12 | |
| VPERM2I128 $0x02, Y13, Y1, Y4 | |
| VPERM2I128 $0x13, Y5, Y9, Y5 | |
| VPERM2I128 $0x13, Y13, Y1, Y9 | |
| sealAVX2ShortSeal: | |
| // Hash aad | |
| MOVQ ad_len+80(FP), R9 | |
| CALL polyHashADInternal<>(SB) | |
| XORQ CX, CX | |
| sealAVX2SealHash: | |
| // itr1 holds the number of bytes encrypted but not yet hashed | |
| CMPQ CX, $0x10 | |
| JB sealAVX2ShortSealLoop | |
| ADDQ (DI), R10 | |
| ADCQ 8(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| SUBQ $0x10, CX | |
| ADDQ $0x10, DI | |
| JMP sealAVX2SealHash | |
| sealAVX2ShortSealLoop: | |
| CMPQ BX, $0x20 | |
| JB sealAVX2ShortTail32 | |
| SUBQ $0x20, BX | |
| // Load for encryption | |
| VPXOR (SI), Y0, Y0 | |
| VMOVDQU Y0, (DI) | |
| LEAQ 32(SI), SI | |
| // Now can hash | |
| ADDQ (DI), R10 | |
| ADCQ 8(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ 16(DI), R10 | |
| ADCQ 24(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 32(DI), DI | |
| // Shift stream left | |
| VMOVDQA Y14, Y0 | |
| VMOVDQA Y12, Y14 | |
| VMOVDQA Y4, Y12 | |
| VMOVDQA Y5, Y4 | |
| VMOVDQA Y9, Y5 | |
| VMOVDQA Y13, Y9 | |
| VMOVDQA Y1, Y13 | |
| VMOVDQA Y6, Y1 | |
| VMOVDQA Y10, Y6 | |
| JMP sealAVX2ShortSealLoop | |
| sealAVX2ShortTail32: | |
| CMPQ BX, $0x10 | |
| VMOVDQA X0, X1 | |
| JB sealAVX2ShortDone | |
| SUBQ $0x10, BX | |
| // Load for encryption | |
| VPXOR (SI), X0, X12 | |
| VMOVDQU X12, (DI) | |
| LEAQ 16(SI), SI | |
| // Hash | |
| ADDQ (DI), R10 | |
| ADCQ 8(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 16(DI), DI | |
| VPERM2I128 $0x11, Y0, Y0, Y0 | |
| VMOVDQA X0, X1 | |
| sealAVX2ShortDone: | |
| VZEROUPPER | |
| JMP sealSSETail | |
| seal320AVX2: | |
| VMOVDQA Y0, Y5 | |
| VMOVDQA Y14, Y9 | |
| VMOVDQA Y12, Y13 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 | |
| VMOVDQA Y0, Y6 | |
| VMOVDQA Y14, Y10 | |
| VMOVDQA Y12, Y8 | |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 | |
| VMOVDQA Y14, Y7 | |
| VMOVDQA Y12, Y11 | |
| VMOVDQA Y4, Y15 | |
| MOVQ $0x0000000a, R9 | |
| sealAVX2320InnerCipherLoop: | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x0c, Y10, Y3 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y3, Y10, Y10 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x07, Y10, Y3 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y3, Y10, Y10 | |
| VPALIGNR $0x04, Y14, Y14, Y14 | |
| VPALIGNR $0x04, Y9, Y9, Y9 | |
| VPALIGNR $0x04, Y10, Y10, Y10 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x0c, Y4, Y4, Y4 | |
| VPALIGNR $0x0c, Y1, Y1, Y1 | |
| VPALIGNR $0x0c, Y2, Y2, Y2 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x0c, Y10, Y3 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y3, Y10, Y10 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x07, Y10, Y3 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y3, Y10, Y10 | |
| VPALIGNR $0x0c, Y14, Y14, Y14 | |
| VPALIGNR $0x0c, Y9, Y9, Y9 | |
| VPALIGNR $0x0c, Y10, Y10, Y10 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x04, Y4, Y4, Y4 | |
| VPALIGNR $0x04, Y1, Y1, Y1 | |
| VPALIGNR $0x04, Y2, Y2, Y2 | |
| DECQ R9 | |
| JNE sealAVX2320InnerCipherLoop | |
| VMOVDQA ·chacha20Constants<>+0(SB), Y3 | |
| VPADDD Y3, Y0, Y0 | |
| VPADDD Y3, Y5, Y5 | |
| VPADDD Y3, Y6, Y6 | |
| VPADDD Y7, Y14, Y14 | |
| VPADDD Y7, Y9, Y9 | |
| VPADDD Y7, Y10, Y10 | |
| VPADDD Y11, Y12, Y12 | |
| VPADDD Y11, Y13, Y13 | |
| VPADDD Y11, Y8, Y8 | |
| VMOVDQA ·avx2IncMask<>+0(SB), Y3 | |
| VPADDD Y15, Y4, Y4 | |
| VPADDD Y3, Y15, Y15 | |
| VPADDD Y15, Y1, Y1 | |
| VPADDD Y3, Y15, Y15 | |
| VPADDD Y15, Y2, Y2 | |
| // Clamp and store poly key | |
| VPERM2I128 $0x02, Y0, Y14, Y3 | |
| VPAND ·polyClampMask<>+0(SB), Y3, Y3 | |
| VMOVDQA Y3, (BP) | |
| // Stream for up to 320 bytes | |
| VPERM2I128 $0x13, Y0, Y14, Y0 | |
| VPERM2I128 $0x13, Y12, Y4, Y14 | |
| VPERM2I128 $0x02, Y5, Y9, Y12 | |
| VPERM2I128 $0x02, Y13, Y1, Y4 | |
| VPERM2I128 $0x13, Y5, Y9, Y5 | |
| VPERM2I128 $0x13, Y13, Y1, Y9 | |
| VPERM2I128 $0x02, Y6, Y10, Y13 | |
| VPERM2I128 $0x02, Y8, Y2, Y1 | |
| VPERM2I128 $0x13, Y6, Y10, Y6 | |
| VPERM2I128 $0x13, Y8, Y2, Y10 | |
| JMP sealAVX2ShortSeal | |
| sealAVX2Tail128: | |
| VMOVDQA ·chacha20Constants<>+0(SB), Y0 | |
| VMOVDQA 32(BP), Y14 | |
| VMOVDQA 64(BP), Y12 | |
| VMOVDQA 192(BP), Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 | |
| VMOVDQA Y4, Y1 | |
| sealAVX2Tail128LoopA: | |
| ADDQ (DI), R10 | |
| ADCQ 8(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 16(DI), DI | |
| sealAVX2Tail128LoopB: | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| ADDQ (DI), R10 | |
| ADCQ 8(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| VPALIGNR $0x04, Y14, Y14, Y14 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x0c, Y4, Y4, Y4 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| ADDQ 16(DI), R10 | |
| ADCQ 24(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 32(DI), DI | |
| VPALIGNR $0x0c, Y14, Y14, Y14 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x04, Y4, Y4, Y4 | |
| DECQ CX | |
| JG sealAVX2Tail128LoopA | |
| DECQ R9 | |
| JGE sealAVX2Tail128LoopB | |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y5 | |
| VPADDD 32(BP), Y14, Y9 | |
| VPADDD 64(BP), Y12, Y13 | |
| VPADDD Y1, Y4, Y1 | |
| VPERM2I128 $0x02, Y5, Y9, Y0 | |
| VPERM2I128 $0x02, Y13, Y1, Y14 | |
| VPERM2I128 $0x13, Y5, Y9, Y12 | |
| VPERM2I128 $0x13, Y13, Y1, Y4 | |
| JMP sealAVX2ShortSealLoop | |
| sealAVX2Tail256: | |
| VMOVDQA ·chacha20Constants<>+0(SB), Y0 | |
| VMOVDQA ·chacha20Constants<>+0(SB), Y5 | |
| VMOVDQA 32(BP), Y14 | |
| VMOVDQA 32(BP), Y9 | |
| VMOVDQA 64(BP), Y12 | |
| VMOVDQA 64(BP), Y13 | |
| VMOVDQA 192(BP), Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 | |
| VMOVDQA Y4, Y7 | |
| VMOVDQA Y1, Y11 | |
| sealAVX2Tail256LoopA: | |
| ADDQ (DI), R10 | |
| ADCQ 8(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 16(DI), DI | |
| sealAVX2Tail256LoopB: | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| ADDQ (DI), R10 | |
| ADCQ 8(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| VPALIGNR $0x04, Y14, Y14, Y14 | |
| VPALIGNR $0x04, Y9, Y9, Y9 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x0c, Y4, Y4, Y4 | |
| VPALIGNR $0x0c, Y1, Y1, Y1 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| ADDQ 16(DI), R10 | |
| ADCQ 24(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 32(DI), DI | |
| VPALIGNR $0x0c, Y14, Y14, Y14 | |
| VPALIGNR $0x0c, Y9, Y9, Y9 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x04, Y4, Y4, Y4 | |
| VPALIGNR $0x04, Y1, Y1, Y1 | |
| DECQ CX | |
| JG sealAVX2Tail256LoopA | |
| DECQ R9 | |
| JGE sealAVX2Tail256LoopB | |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 | |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 | |
| VPADDD 32(BP), Y14, Y14 | |
| VPADDD 32(BP), Y9, Y9 | |
| VPADDD 64(BP), Y12, Y12 | |
| VPADDD 64(BP), Y13, Y13 | |
| VPADDD Y7, Y4, Y4 | |
| VPADDD Y11, Y1, Y1 | |
| VPERM2I128 $0x02, Y0, Y14, Y3 | |
| VPERM2I128 $0x02, Y12, Y4, Y7 | |
| VPERM2I128 $0x13, Y0, Y14, Y11 | |
| VPERM2I128 $0x13, Y12, Y4, Y15 | |
| VPXOR (SI), Y3, Y3 | |
| VPXOR 32(SI), Y7, Y7 | |
| VPXOR 64(SI), Y11, Y11 | |
| VPXOR 96(SI), Y15, Y15 | |
| VMOVDQU Y3, (DI) | |
| VMOVDQU Y7, 32(DI) | |
| VMOVDQU Y11, 64(DI) | |
| VMOVDQU Y15, 96(DI) | |
| MOVQ $0x00000080, CX | |
| LEAQ 128(SI), SI | |
| SUBQ $0x80, BX | |
| VPERM2I128 $0x02, Y5, Y9, Y0 | |
| VPERM2I128 $0x02, Y13, Y1, Y14 | |
| VPERM2I128 $0x13, Y5, Y9, Y12 | |
| VPERM2I128 $0x13, Y13, Y1, Y4 | |
| JMP sealAVX2SealHash | |
| sealAVX2Tail384: | |
| VMOVDQA ·chacha20Constants<>+0(SB), Y0 | |
| VMOVDQA Y0, Y5 | |
| VMOVDQA Y0, Y6 | |
| VMOVDQA 32(BP), Y14 | |
| VMOVDQA Y14, Y9 | |
| VMOVDQA Y14, Y10 | |
| VMOVDQA 64(BP), Y12 | |
| VMOVDQA Y12, Y13 | |
| VMOVDQA Y12, Y8 | |
| VMOVDQA 192(BP), Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 | |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 | |
| VMOVDQA Y4, Y7 | |
| VMOVDQA Y1, Y11 | |
| VMOVDQA Y2, Y15 | |
| sealAVX2Tail384LoopA: | |
| ADDQ (DI), R10 | |
| ADCQ 8(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 16(DI), DI | |
| sealAVX2Tail384LoopB: | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x0c, Y10, Y3 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y3, Y10, Y10 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x07, Y10, Y3 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y3, Y10, Y10 | |
| ADDQ (DI), R10 | |
| ADCQ 8(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| VPALIGNR $0x04, Y14, Y14, Y14 | |
| VPALIGNR $0x04, Y9, Y9, Y9 | |
| VPALIGNR $0x04, Y10, Y10, Y10 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x0c, Y4, Y4, Y4 | |
| VPALIGNR $0x0c, Y1, Y1, Y1 | |
| VPALIGNR $0x0c, Y2, Y2, Y2 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x0c, Y14, Y3 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y14, Y0, Y0 | |
| VPXOR Y0, Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPADDD Y4, Y12, Y12 | |
| VPXOR Y12, Y14, Y14 | |
| VPSLLD $0x07, Y14, Y3 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y3, Y14, Y14 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x0c, Y9, Y3 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y9, Y5, Y5 | |
| VPXOR Y5, Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPADDD Y1, Y13, Y13 | |
| VPXOR Y13, Y9, Y9 | |
| VPSLLD $0x07, Y9, Y3 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y3, Y9, Y9 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x0c, Y10, Y3 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y3, Y10, Y10 | |
| VPADDD Y10, Y6, Y6 | |
| VPXOR Y6, Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPADDD Y2, Y8, Y8 | |
| VPXOR Y8, Y10, Y10 | |
| VPSLLD $0x07, Y10, Y3 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y3, Y10, Y10 | |
| ADDQ 16(DI), R10 | |
| ADCQ 24(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 32(DI), DI | |
| VPALIGNR $0x0c, Y14, Y14, Y14 | |
| VPALIGNR $0x0c, Y9, Y9, Y9 | |
| VPALIGNR $0x0c, Y10, Y10, Y10 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x04, Y4, Y4, Y4 | |
| VPALIGNR $0x04, Y1, Y1, Y1 | |
| VPALIGNR $0x04, Y2, Y2, Y2 | |
| DECQ CX | |
| JG sealAVX2Tail384LoopA | |
| DECQ R9 | |
| JGE sealAVX2Tail384LoopB | |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 | |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 | |
| VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 | |
| VPADDD 32(BP), Y14, Y14 | |
| VPADDD 32(BP), Y9, Y9 | |
| VPADDD 32(BP), Y10, Y10 | |
| VPADDD 64(BP), Y12, Y12 | |
| VPADDD 64(BP), Y13, Y13 | |
| VPADDD 64(BP), Y8, Y8 | |
| VPADDD Y7, Y4, Y4 | |
| VPADDD Y11, Y1, Y1 | |
| VPADDD Y15, Y2, Y2 | |
| VPERM2I128 $0x02, Y0, Y14, Y3 | |
| VPERM2I128 $0x02, Y12, Y4, Y7 | |
| VPERM2I128 $0x13, Y0, Y14, Y11 | |
| VPERM2I128 $0x13, Y12, Y4, Y15 | |
| VPXOR (SI), Y3, Y3 | |
| VPXOR 32(SI), Y7, Y7 | |
| VPXOR 64(SI), Y11, Y11 | |
| VPXOR 96(SI), Y15, Y15 | |
| VMOVDQU Y3, (DI) | |
| VMOVDQU Y7, 32(DI) | |
| VMOVDQU Y11, 64(DI) | |
| VMOVDQU Y15, 96(DI) | |
| VPERM2I128 $0x02, Y5, Y9, Y3 | |
| VPERM2I128 $0x02, Y13, Y1, Y7 | |
| VPERM2I128 $0x13, Y5, Y9, Y11 | |
| VPERM2I128 $0x13, Y13, Y1, Y15 | |
| VPXOR 128(SI), Y3, Y3 | |
| VPXOR 160(SI), Y7, Y7 | |
| VPXOR 192(SI), Y11, Y11 | |
| VPXOR 224(SI), Y15, Y15 | |
| VMOVDQU Y3, 128(DI) | |
| VMOVDQU Y7, 160(DI) | |
| VMOVDQU Y11, 192(DI) | |
| VMOVDQU Y15, 224(DI) | |
| MOVQ $0x00000100, CX | |
| LEAQ 256(SI), SI | |
| SUBQ $0x00000100, BX | |
| VPERM2I128 $0x02, Y6, Y10, Y0 | |
| VPERM2I128 $0x02, Y8, Y2, Y14 | |
| VPERM2I128 $0x13, Y6, Y10, Y12 | |
| VPERM2I128 $0x13, Y8, Y2, Y4 | |
| JMP sealAVX2SealHash | |
| sealAVX2Tail512: | |
| VMOVDQA ·chacha20Constants<>+0(SB), Y0 | |
| VMOVDQA Y0, Y5 | |
| VMOVDQA Y0, Y6 | |
| VMOVDQA Y0, Y7 | |
| VMOVDQA 32(BP), Y14 | |
| VMOVDQA Y14, Y9 | |
| VMOVDQA Y14, Y10 | |
| VMOVDQA Y14, Y11 | |
| VMOVDQA 64(BP), Y12 | |
| VMOVDQA Y12, Y13 | |
| VMOVDQA Y12, Y8 | |
| VMOVDQA Y12, Y15 | |
| VMOVDQA 192(BP), Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 | |
| VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 | |
| VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 | |
| VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 | |
| VMOVDQA Y4, 96(BP) | |
| VMOVDQA Y1, 128(BP) | |
| VMOVDQA Y2, 160(BP) | |
| VMOVDQA Y3, 192(BP) | |
| sealAVX2Tail512LoopA: | |
| ADDQ (DI), R10 | |
| ADCQ 8(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), AX | |
| MOVQ AX, R15 | |
| MULQ R10 | |
| MOVQ AX, R13 | |
| MOVQ DX, R14 | |
| MOVQ (BP), AX | |
| MULQ R11 | |
| IMULQ R12, R15 | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), AX | |
| MOVQ AX, R8 | |
| MULQ R10 | |
| ADDQ AX, R14 | |
| ADCQ $0x00, DX | |
| MOVQ DX, R10 | |
| MOVQ 8(BP), AX | |
| MULQ R11 | |
| ADDQ AX, R15 | |
| ADCQ $0x00, DX | |
| IMULQ R12, R8 | |
| ADDQ R10, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 16(DI), DI | |
| sealAVX2Tail512LoopB: | |
| VPADDD Y14, Y0, Y0 | |
| VPADDD Y9, Y5, Y5 | |
| VPADDD Y10, Y6, Y6 | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y0, Y4, Y4 | |
| VPXOR Y5, Y1, Y1 | |
| VPXOR Y6, Y2, Y2 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 | |
| VPADDD Y4, Y12, Y12 | |
| VPADDD Y1, Y13, Y13 | |
| VPADDD Y2, Y8, Y8 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y12, Y14, Y14 | |
| VPXOR Y13, Y9, Y9 | |
| VPXOR Y8, Y10, Y10 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA Y15, 224(BP) | |
| VPSLLD $0x0c, Y14, Y15 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPSLLD $0x0c, Y9, Y15 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPSLLD $0x0c, Y10, Y15 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPSLLD $0x0c, Y11, Y15 | |
| VPSRLD $0x14, Y11, Y11 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA 224(BP), Y15 | |
| ADDQ (DI), R10 | |
| ADCQ 8(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| VPADDD Y14, Y0, Y0 | |
| VPADDD Y9, Y5, Y5 | |
| VPADDD Y10, Y6, Y6 | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y0, Y4, Y4 | |
| VPXOR Y5, Y1, Y1 | |
| VPXOR Y6, Y2, Y2 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 | |
| VPADDD Y4, Y12, Y12 | |
| VPADDD Y1, Y13, Y13 | |
| VPADDD Y2, Y8, Y8 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y12, Y14, Y14 | |
| VPXOR Y13, Y9, Y9 | |
| VPXOR Y8, Y10, Y10 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA Y15, 224(BP) | |
| VPSLLD $0x07, Y14, Y15 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPSLLD $0x07, Y9, Y15 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPSLLD $0x07, Y10, Y15 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPSLLD $0x07, Y11, Y15 | |
| VPSRLD $0x19, Y11, Y11 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA 224(BP), Y15 | |
| VPALIGNR $0x04, Y14, Y14, Y14 | |
| VPALIGNR $0x04, Y9, Y9, Y9 | |
| VPALIGNR $0x04, Y10, Y10, Y10 | |
| VPALIGNR $0x04, Y11, Y11, Y11 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x08, Y15, Y15, Y15 | |
| VPALIGNR $0x0c, Y4, Y4, Y4 | |
| VPALIGNR $0x0c, Y1, Y1, Y1 | |
| VPALIGNR $0x0c, Y2, Y2, Y2 | |
| VPALIGNR $0x0c, Y3, Y3, Y3 | |
| VPADDD Y14, Y0, Y0 | |
| VPADDD Y9, Y5, Y5 | |
| VPADDD Y10, Y6, Y6 | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y0, Y4, Y4 | |
| VPXOR Y5, Y1, Y1 | |
| VPXOR Y6, Y2, Y2 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol16<>+0(SB), Y4, Y4 | |
| VPSHUFB ·rol16<>+0(SB), Y1, Y1 | |
| VPSHUFB ·rol16<>+0(SB), Y2, Y2 | |
| VPSHUFB ·rol16<>+0(SB), Y3, Y3 | |
| VPADDD Y4, Y12, Y12 | |
| VPADDD Y1, Y13, Y13 | |
| VPADDD Y2, Y8, Y8 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y12, Y14, Y14 | |
| VPXOR Y13, Y9, Y9 | |
| VPXOR Y8, Y10, Y10 | |
| VPXOR Y15, Y11, Y11 | |
| ADDQ 16(DI), R10 | |
| ADCQ 24(DI), R11 | |
| ADCQ $0x01, R12 | |
| MOVQ (BP), DX | |
| MOVQ DX, R15 | |
| MULXQ R10, R13, R14 | |
| IMULQ R12, R15 | |
| MULXQ R11, AX, DX | |
| ADDQ AX, R14 | |
| ADCQ DX, R15 | |
| MOVQ 8(BP), DX | |
| MULXQ R10, R10, AX | |
| ADDQ R10, R14 | |
| MULXQ R11, R11, R8 | |
| ADCQ R11, R15 | |
| ADCQ $0x00, R8 | |
| IMULQ R12, DX | |
| ADDQ AX, R15 | |
| ADCQ DX, R8 | |
| MOVQ R13, R10 | |
| MOVQ R14, R11 | |
| MOVQ R15, R12 | |
| ANDQ $0x03, R12 | |
| MOVQ R15, R13 | |
| ANDQ $-4, R13 | |
| MOVQ R8, R14 | |
| SHRQ $0x02, R8, R15 | |
| SHRQ $0x02, R8 | |
| ADDQ R13, R10 | |
| ADCQ R14, R11 | |
| ADCQ $0x00, R12 | |
| ADDQ R15, R10 | |
| ADCQ R8, R11 | |
| ADCQ $0x00, R12 | |
| LEAQ 32(DI), DI | |
| VMOVDQA Y15, 224(BP) | |
| VPSLLD $0x0c, Y14, Y15 | |
| VPSRLD $0x14, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPSLLD $0x0c, Y9, Y15 | |
| VPSRLD $0x14, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPSLLD $0x0c, Y10, Y15 | |
| VPSRLD $0x14, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPSLLD $0x0c, Y11, Y15 | |
| VPSRLD $0x14, Y11, Y11 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA 224(BP), Y15 | |
| VPADDD Y14, Y0, Y0 | |
| VPADDD Y9, Y5, Y5 | |
| VPADDD Y10, Y6, Y6 | |
| VPADDD Y11, Y7, Y7 | |
| VPXOR Y0, Y4, Y4 | |
| VPXOR Y5, Y1, Y1 | |
| VPXOR Y6, Y2, Y2 | |
| VPXOR Y7, Y3, Y3 | |
| VPSHUFB ·rol8<>+0(SB), Y4, Y4 | |
| VPSHUFB ·rol8<>+0(SB), Y1, Y1 | |
| VPSHUFB ·rol8<>+0(SB), Y2, Y2 | |
| VPSHUFB ·rol8<>+0(SB), Y3, Y3 | |
| VPADDD Y4, Y12, Y12 | |
| VPADDD Y1, Y13, Y13 | |
| VPADDD Y2, Y8, Y8 | |
| VPADDD Y3, Y15, Y15 | |
| VPXOR Y12, Y14, Y14 | |
| VPXOR Y13, Y9, Y9 | |
| VPXOR Y8, Y10, Y10 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA Y15, 224(BP) | |
| VPSLLD $0x07, Y14, Y15 | |
| VPSRLD $0x19, Y14, Y14 | |
| VPXOR Y15, Y14, Y14 | |
| VPSLLD $0x07, Y9, Y15 | |
| VPSRLD $0x19, Y9, Y9 | |
| VPXOR Y15, Y9, Y9 | |
| VPSLLD $0x07, Y10, Y15 | |
| VPSRLD $0x19, Y10, Y10 | |
| VPXOR Y15, Y10, Y10 | |
| VPSLLD $0x07, Y11, Y15 | |
| VPSRLD $0x19, Y11, Y11 | |
| VPXOR Y15, Y11, Y11 | |
| VMOVDQA 224(BP), Y15 | |
| VPALIGNR $0x0c, Y14, Y14, Y14 | |
| VPALIGNR $0x0c, Y9, Y9, Y9 | |
| VPALIGNR $0x0c, Y10, Y10, Y10 | |
| VPALIGNR $0x0c, Y11, Y11, Y11 | |
| VPALIGNR $0x08, Y12, Y12, Y12 | |
| VPALIGNR $0x08, Y13, Y13, Y13 | |
| VPALIGNR $0x08, Y8, Y8, Y8 | |
| VPALIGNR $0x08, Y15, Y15, Y15 | |
| VPALIGNR $0x04, Y4, Y4, Y4 | |
| VPALIGNR $0x04, Y1, Y1, Y1 | |
| VPALIGNR $0x04, Y2, Y2, Y2 | |
| VPALIGNR $0x04, Y3, Y3, Y3 | |
| DECQ CX | |
| JG sealAVX2Tail512LoopA | |
| DECQ R9 | |
| JGE sealAVX2Tail512LoopB | |
| VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 | |
| VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 | |
| VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 | |
| VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 | |
| VPADDD 32(BP), Y14, Y14 | |
| VPADDD 32(BP), Y9, Y9 | |
| VPADDD 32(BP), Y10, Y10 | |
| VPADDD 32(BP), Y11, Y11 | |
| VPADDD 64(BP), Y12, Y12 | |
| VPADDD 64(BP), Y13, Y13 | |
| VPADDD 64(BP), Y8, Y8 | |
| VPADDD 64(BP), Y15, Y15 | |
| VPADDD 96(BP), Y4, Y4 | |
| VPADDD 128(BP), Y1, Y1 | |
| VPADDD 160(BP), Y2, Y2 | |
| VPADDD 192(BP), Y3, Y3 | |
| VMOVDQA Y15, 224(BP) | |
| VPERM2I128 $0x02, Y0, Y14, Y15 | |
| VPXOR (SI), Y15, Y15 | |
| VMOVDQU Y15, (DI) | |
| VPERM2I128 $0x02, Y12, Y4, Y15 | |
| VPXOR 32(SI), Y15, Y15 | |
| VMOVDQU Y15, 32(DI) | |
| VPERM2I128 $0x13, Y0, Y14, Y15 | |
| VPXOR 64(SI), Y15, Y15 | |
| VMOVDQU Y15, 64(DI) | |
| VPERM2I128 $0x13, Y12, Y4, Y15 | |
| VPXOR 96(SI), Y15, Y15 | |
| VMOVDQU Y15, 96(DI) | |
| VPERM2I128 $0x02, Y5, Y9, Y0 | |
| VPERM2I128 $0x02, Y13, Y1, Y14 | |
| VPERM2I128 $0x13, Y5, Y9, Y12 | |
| VPERM2I128 $0x13, Y13, Y1, Y4 | |
| VPXOR 128(SI), Y0, Y0 | |
| VPXOR 160(SI), Y14, Y14 | |
| VPXOR 192(SI), Y12, Y12 | |
| VPXOR 224(SI), Y4, Y4 | |
| VMOVDQU Y0, 128(DI) | |
| VMOVDQU Y14, 160(DI) | |
| VMOVDQU Y12, 192(DI) | |
| VMOVDQU Y4, 224(DI) | |
| VPERM2I128 $0x02, Y6, Y10, Y0 | |
| VPERM2I128 $0x02, Y8, Y2, Y14 | |
| VPERM2I128 $0x13, Y6, Y10, Y12 | |
| VPERM2I128 $0x13, Y8, Y2, Y4 | |
| VPXOR 256(SI), Y0, Y0 | |
| VPXOR 288(SI), Y14, Y14 | |
| VPXOR 320(SI), Y12, Y12 | |
| VPXOR 352(SI), Y4, Y4 | |
| VMOVDQU Y0, 256(DI) | |
| VMOVDQU Y14, 288(DI) | |
| VMOVDQU Y12, 320(DI) | |
| VMOVDQU Y4, 352(DI) | |
| MOVQ $0x00000180, CX | |
| LEAQ 384(SI), SI | |
| SUBQ $0x00000180, BX | |
| VPERM2I128 $0x02, Y7, Y11, Y0 | |
| VPERM2I128 $0x02, 224(BP), Y3, Y14 | |
| VPERM2I128 $0x13, Y7, Y11, Y12 | |
| VPERM2I128 $0x13, 224(BP), Y3, Y4 | |
| JMP sealAVX2SealHash |