// Code generated by command: go run chacha20poly1305_amd64_asm.go -out ../chacha20poly1305_amd64.s -pkg chacha20poly1305. DO NOT EDIT.

//go:build gc && !purego

#include "textflag.h"

// func polyHashADInternal<>()
TEXT polyHashADInternal<>(SB), NOSPLIT, $0
	// Hack: Must declare #define macros inside of a function due to Avo constraints
	// ROL rotates the uint32s in register R left by N bits, using temporary T.
	#define ROL(N, R, T) \
		MOVO R, T; \
		PSLLL $(N), T; \
		PSRLL $(32-(N)), R; \
		PXOR T, R

	// ROL8 rotates the uint32s in register R left by 8, using temporary T if needed.
	#ifdef GOAMD64_v2
		#define ROL8(R, T) PSHUFB ·rol8<>(SB), R
	#else
		#define ROL8(R, T) ROL(8, R, T)
	#endif

	// ROL16 rotates the uint32s in register R left by 16, using temporary T if needed.
	#ifdef GOAMD64_v2
		#define ROL16(R, T) PSHUFB ·rol16<>(SB), R
	#else
		#define ROL16(R, T) ROL(16, R, T)
	#endif
	XORQ  R10, R10
	XORQ  R11, R11
	XORQ  R12, R12
	CMPQ  R9, $0x0d
	JNE   hashADLoop
	MOVQ  (CX), R10
	MOVQ  5(CX), R11
	SHRQ  $0x18, R11
	MOVQ  $0x00000001, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	RET

hashADLoop:
	// Hash in 16 byte chunks
	CMPQ  R9, $0x10
	JB    hashADTail
	ADDQ  (CX), R10
	ADCQ  8(CX), R11
	ADCQ  $0x01, R12
	LEAQ  16(CX), CX
	SUBQ  $0x10, R9
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	JMP   hashADLoop

hashADTail:
	CMPQ R9, $0x00
	JE   hashADDone

	// Hash last < 16 byte tail
	XORQ R13, R13
	XORQ R14, R14
	XORQ R15, R15
	ADDQ R9, CX

hashADTailLoop:
	SHLQ  $0x08, R13, R14
	SHLQ  $0x08, R13
	MOVB  -1(CX), R15
	XORQ  R15, R13
	DECQ  CX
	DECQ  R9
	JNE   hashADTailLoop
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12

hashADDone:
	RET

// func chacha20Poly1305Open(dst []byte, key []uint32, src []byte, ad []byte) bool
// Requires: AVX, AVX2, BMI2, CMOV, SSE2
TEXT ·chacha20Poly1305Open(SB), $288-97
	// For aligned stack access
	MOVQ SP, BP
	ADDQ $0x20, BP
	ANDQ $-32, BP
	MOVQ dst_base+0(FP), DI
	MOVQ key_base+24(FP), R8
	MOVQ src_base+48(FP), SI
	MOVQ src_len+56(FP), BX
	MOVQ ad_base+72(FP), CX

	// Check for AVX2 support
	CMPB ·useAVX2+0(SB), $0x01
	JE   chacha20Poly1305Open_AVX2

	// Special optimization, for very short buffers
	CMPQ BX, $0x80
	JBE  openSSE128

	// For long buffers, prepare the poly key first
	MOVOU ·chacha20Constants<>+0(SB), X0
	MOVOU 16(R8), X3
	MOVOU 32(R8), X6
	MOVOU 48(R8), X9
	MOVO  X9, X13

	// Store state on stack for future use
	MOVO X3, 32(BP)
	MOVO X6, 48(BP)
	MOVO X9, 128(BP)
	MOVQ $0x0000000a, R9

openSSEPreparePolyKey:
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X3
	PXOR  X12, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X3
	PXOR  X12, X3
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x0c
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X3
	PXOR  X12, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X3
	PXOR  X12, X3
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x04
	DECQ  R9
	JNE   openSSEPreparePolyKey

	// A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded
	PADDL ·chacha20Constants<>+0(SB), X0
	PADDL 32(BP), X3

	// Clamp and store the key
	PAND ·polyClampMask<>+0(SB), X0
	MOVO X0, (BP)
	MOVO X3, 16(BP)

	// Hash AAD
	MOVQ ad_len+80(FP), R9
	CALL polyHashADInternal<>(SB)

openSSEMainLoop:
	CMPQ BX, $0x00000100
	JB   openSSEMainLoopDone

	// Load state, increment counter blocks
	MOVO  ·chacha20Constants<>+0(SB), X0
	MOVO  32(BP), X3
	MOVO  48(BP), X6
	MOVO  128(BP), X9
	PADDL ·sseIncMask<>+0(SB), X9
	MOVO  X0, X1
	MOVO  X3, X4
	MOVO  X6, X7
	MOVO  X9, X10
	PADDL ·sseIncMask<>+0(SB), X10
	MOVO  X1, X2
	MOVO  X4, X5
	MOVO  X7, X8
	MOVO  X10, X11
	PADDL ·sseIncMask<>+0(SB), X11
	MOVO  X2, X12
	MOVO  X5, X13
	MOVO  X8, X14
	MOVO  X11, X15
	PADDL ·sseIncMask<>+0(SB), X15

	// Store counters
	MOVO X9, 80(BP)
	MOVO X10, 96(BP)
	MOVO X11, 112(BP)
	MOVO X15, 128(BP)

	// There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we hash
	// 2 blocks, and for the remaining 4 only 1 block - for a total of 16
	MOVQ $0x00000004, CX
	MOVQ SI, R9

openSSEInternalLoop:
	MOVO  X14, 64(BP)
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X14)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X3
	PXOR  X14, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X14)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X3
	PXOR  X14, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X14)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X4
	PXOR  X14, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X14)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X4
	PXOR  X14, X4
	PADDD X5, X2
	PXOR  X2, X11
	ROL16(X11, X14)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X5
	PXOR  X14, X5
	PADDD X5, X2
	PXOR  X2, X11
	ROL8(X11, X14)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X5
	PXOR  X14, X5
	MOVO  64(BP), X14
	MOVO  X7, 64(BP)
	PADDD X13, X12
	PXOR  X12, X15
	ROL16(X15, X7)
	PADDD X15, X14
	PXOR  X14, X13
	MOVO  X13, X7
	PSLLL $0x0c, X7
	PSRLL $0x14, X13
	PXOR  X7, X13
	PADDD X13, X12
	PXOR  X12, X15
	ROL8(X15, X7)
	PADDD X15, X14
	PXOR  X14, X13
	MOVO  X13, X7
	PSLLL $0x07, X7
	PSRLL $0x19, X13
	PXOR  X7, X13
	MOVO  64(BP), X7
	ADDQ  (R9), R10
	ADCQ  8(R9), R11
	ADCQ  $0x01, R12
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc0
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x0c
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	LEAQ  16(R9), R9
	MOVO  X14, 64(BP)
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X14)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X3
	PXOR  X14, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X14)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X3
	PXOR  X14, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X14)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X4
	PXOR  X14, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X14)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X4
	PXOR  X14, X4
	PADDD X5, X2
	PXOR  X2, X11
	ROL16(X11, X14)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X5
	PXOR  X14, X5
	PADDD X5, X2
	PXOR  X2, X11
	ROL8(X11, X14)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X5
	PXOR  X14, X5
	MOVO  64(BP), X14
	MOVO  X7, 64(BP)
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	PADDD X13, X12
	PXOR  X12, X15
	ROL16(X15, X7)
	PADDD X15, X14
	PXOR  X14, X13
	MOVO  X13, X7
	PSLLL $0x0c, X7
	PSRLL $0x14, X13
	PXOR  X7, X13
	PADDD X13, X12
	PXOR  X12, X15
	ROL8(X15, X7)
	PADDD X15, X14
	PXOR  X14, X13
	MOVO  X13, X7
	PSLLL $0x07, X7
	PSRLL $0x19, X13
	PXOR  X7, X13
	MOVO  64(BP), X7
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc0
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x04
	DECQ  CX
	JGE   openSSEInternalLoop
	ADDQ  (R9), R10
	ADCQ  8(R9), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(R9), R9
	CMPQ  CX, $-6
	JG    openSSEInternalLoop

	// Add in the state
	PADDD ·chacha20Constants<>+0(SB), X0
	PADDD ·chacha20Constants<>+0(SB), X1
	PADDD ·chacha20Constants<>+0(SB), X2
	PADDD ·chacha20Constants<>+0(SB), X12
	PADDD 32(BP), X3
	PADDD 32(BP), X4
	PADDD 32(BP), X5
	PADDD 32(BP), X13
	PADDD 48(BP), X6
	PADDD 48(BP), X7
	PADDD 48(BP), X8
	PADDD 48(BP), X14
	PADDD 80(BP), X9
	PADDD 96(BP), X10
	PADDD 112(BP), X11
	PADDD 128(BP), X15

	// Load - xor - store
	MOVO  X15, 64(BP)
	MOVOU (SI), X15
	PXOR  X15, X0
	MOVOU X0, (DI)
	MOVOU 16(SI), X15
	PXOR  X15, X3
	MOVOU X3, 16(DI)
	MOVOU 32(SI), X15
	PXOR  X15, X6
	MOVOU X6, 32(DI)
	MOVOU 48(SI), X15
	PXOR  X15, X9
	MOVOU X9, 48(DI)
	MOVOU 64(SI), X9
	PXOR  X9, X1
	MOVOU X1, 64(DI)
	MOVOU 80(SI), X9
	PXOR  X9, X4
	MOVOU X4, 80(DI)
	MOVOU 96(SI), X9
	PXOR  X9, X7
	MOVOU X7, 96(DI)
	MOVOU 112(SI), X9
	PXOR  X9, X10
	MOVOU X10, 112(DI)
	MOVOU 128(SI), X9
	PXOR  X9, X2
	MOVOU X2, 128(DI)
	MOVOU 144(SI), X9
	PXOR  X9, X5
	MOVOU X5, 144(DI)
	MOVOU 160(SI), X9
	PXOR  X9, X8
	MOVOU X8, 160(DI)
	MOVOU 176(SI), X9
	PXOR  X9, X11
	MOVOU X11, 176(DI)
	MOVOU 192(SI), X9
	PXOR  X9, X12
	MOVOU X12, 192(DI)
	MOVOU 208(SI), X9
	PXOR  X9, X13
	MOVOU X13, 208(DI)
	MOVOU 224(SI), X9
	PXOR  X9, X14
	MOVOU X14, 224(DI)
	MOVOU 240(SI), X9
	PXOR  64(BP), X9
	MOVOU X9, 240(DI)
	LEAQ  256(SI), SI
	LEAQ  256(DI), DI
	SUBQ  $0x00000100, BX
	JMP   openSSEMainLoop

openSSEMainLoopDone:
	// Handle the various tail sizes efficiently
	TESTQ BX, BX
	JE    openSSEFinalize
	CMPQ  BX, $0x40
	JBE   openSSETail64
	CMPQ  BX, $0x80
	JBE   openSSETail128
	CMPQ  BX, $0xc0
	JBE   openSSETail192
	JMP   openSSETail256

openSSEFinalize:
	// Hash in the PT, AAD lengths
	ADDQ  ad_len+80(FP), R10
	ADCQ  src_len+56(FP), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12

	// Final reduce
	MOVQ    R10, R13
	MOVQ    R11, R14
	MOVQ    R12, R15
	SUBQ    $-5, R10
	SBBQ    $-1, R11
	SBBQ    $0x03, R12
	CMOVQCS R13, R10
	CMOVQCS R14, R11
	CMOVQCS R15, R12

	// Add in the "s" part of the key
	ADDQ 16(BP), R10
	ADCQ 24(BP), R11

	// Finally, constant time compare to the tag at the end of the message
	XORQ    AX, AX
	MOVQ    $0x00000001, DX
	XORQ    (SI), R10
	XORQ    8(SI), R11
	ORQ     R11, R10
	CMOVQEQ DX, AX

	// Return true iff tags are equal
	MOVB AX, ret+96(FP)
	RET

openSSE128:
	MOVOU ·chacha20Constants<>+0(SB), X0
	MOVOU 16(R8), X3
	MOVOU 32(R8), X6
	MOVOU 48(R8), X9
	MOVO  X0, X1
	MOVO  X3, X4
	MOVO  X6, X7
	MOVO  X9, X10
	PADDL ·sseIncMask<>+0(SB), X10
	MOVO  X1, X2
	MOVO  X4, X5
	MOVO  X7, X8
	MOVO  X10, X11
	PADDL ·sseIncMask<>+0(SB), X11
	MOVO  X3, X13
	MOVO  X6, X14
	MOVO  X10, X15
	MOVQ  $0x0000000a, R9

openSSE128InnerCipherLoop:
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X3
	PXOR  X12, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X3
	PXOR  X12, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X4
	PXOR  X12, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X4
	PXOR  X12, X4
	PADDD X5, X2
	PXOR  X2, X11
	ROL16(X11, X12)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X5
	PXOR  X12, X5
	PADDD X5, X2
	PXOR  X2, X11
	ROL8(X11, X12)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X5
	PXOR  X12, X5
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc0
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X3
	PXOR  X12, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X3
	PXOR  X12, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X4
	PXOR  X12, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X4
	PXOR  X12, X4
	PADDD X5, X2
	PXOR  X2, X11
	ROL16(X11, X12)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X5
	PXOR  X12, X5
	PADDD X5, X2
	PXOR  X2, X11
	ROL8(X11, X12)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X5
	PXOR  X12, X5
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc0
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	DECQ  R9
	JNE   openSSE128InnerCipherLoop

	// A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded
	PADDL ·chacha20Constants<>+0(SB), X0
	PADDL ·chacha20Constants<>+0(SB), X1
	PADDL ·chacha20Constants<>+0(SB), X2
	PADDL X13, X3
	PADDL X13, X4
	PADDL X13, X5
	PADDL X14, X7
	PADDL X14, X8
	PADDL X15, X10
	PADDL ·sseIncMask<>+0(SB), X15
	PADDL X15, X11

	// Clamp and store the key
	PAND  ·polyClampMask<>+0(SB), X0
	MOVOU X0, (BP)
	MOVOU X3, 16(BP)

	// Hash
	MOVQ ad_len+80(FP), R9
	CALL polyHashADInternal<>(SB)

openSSE128Open:
	CMPQ BX, $0x10
	JB   openSSETail16
	SUBQ $0x10, BX

	// Load for hashing
	ADDQ (SI), R10
	ADCQ 8(SI), R11
	ADCQ $0x01, R12

	// Load for decryption
	MOVOU (SI), X12
	PXOR  X12, X1
	MOVOU X1, (DI)
	LEAQ  16(SI), SI
	LEAQ  16(DI), DI
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12

	// Shift the stream "left"
	MOVO X4, X1
	MOVO X7, X4
	MOVO X10, X7
	MOVO X2, X10
	MOVO X5, X2
	MOVO X8, X5
	MOVO X11, X8
	JMP  openSSE128Open

openSSETail16:
	TESTQ BX, BX
	JE    openSSEFinalize

	// We can safely load the CT from the end, because it is padded with the MAC
	MOVQ  BX, R9
	SHLQ  $0x04, R9
	LEAQ  ·andMask<>+0(SB), R13
	MOVOU (SI), X12
	ADDQ  BX, SI
	PAND  -16(R13)(R9*1), X12
	MOVO  X12, 64(BP)
	MOVQ  X12, R13
	MOVQ  72(BP), R14
	PXOR  X1, X12

	// We can only store one byte at a time, since plaintext can be shorter than 16 bytes
openSSETail16Store:
	MOVQ   X12, R8
	MOVB   R8, (DI)
	PSRLDQ $0x01, X12
	INCQ   DI
	DECQ   BX
	JNE    openSSETail16Store
	ADDQ   R13, R10
	ADCQ   R14, R11
	ADCQ   $0x01, R12
	MOVQ   (BP), AX
	MOVQ   AX, R15
	MULQ   R10
	MOVQ   AX, R13
	MOVQ   DX, R14
	MOVQ   (BP), AX
	MULQ   R11
	IMULQ  R12, R15
	ADDQ   AX, R14
	ADCQ   DX, R15
	MOVQ   8(BP), AX
	MOVQ   AX, R8
	MULQ   R10
	ADDQ   AX, R14
	ADCQ   $0x00, DX
	MOVQ   DX, R10
	MOVQ   8(BP), AX
	MULQ   R11
	ADDQ   AX, R15
	ADCQ   $0x00, DX
	IMULQ  R12, R8
	ADDQ   R10, R15
	ADCQ   DX, R8
	MOVQ   R13, R10
	MOVQ   R14, R11
	MOVQ   R15, R12
	ANDQ   $0x03, R12
	MOVQ   R15, R13
	ANDQ   $-4, R13
	MOVQ   R8, R14
	SHRQ   $0x02, R8, R15
	SHRQ   $0x02, R8
	ADDQ   R13, R10
	ADCQ   R14, R11
	ADCQ   $0x00, R12
	ADDQ   R15, R10
	ADCQ   R8, R11
	ADCQ   $0x00, R12
	JMP    openSSEFinalize

openSSETail64:
	MOVO  ·chacha20Constants<>+0(SB), X0
	MOVO  32(BP), X3
	MOVO  48(BP), X6
	MOVO  128(BP), X9
	PADDL ·sseIncMask<>+0(SB), X9
	MOVO  X9, 80(BP)
	XORQ  R9, R9
	MOVQ  BX, CX
	CMPQ  CX, $0x10
	JB    openSSETail64LoopB

openSSETail64LoopA:
	ADDQ  (SI)(R9*1), R10
	ADCQ  8(SI)(R9*1), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	SUBQ  $0x10, CX

openSSETail64LoopB:
	ADDQ  $0x10, R9
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X3
	PXOR  X12, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X3
	PXOR  X12, X3
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x0c
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X3
	PXOR  X12, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X3
	PXOR  X12, X3
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x04
	CMPQ  CX, $0x10
	JAE   openSSETail64LoopA
	CMPQ  R9, $0xa0
	JNE   openSSETail64LoopB
	PADDL ·chacha20Constants<>+0(SB), X0
	PADDL 32(BP), X3
	PADDL 48(BP), X6
	PADDL 80(BP), X9

openSSETail64DecLoop:
	CMPQ  BX, $0x10
	JB    openSSETail64DecLoopDone
	SUBQ  $0x10, BX
	MOVOU (SI), X12
	PXOR  X12, X0
	MOVOU X0, (DI)
	LEAQ  16(SI), SI
	LEAQ  16(DI), DI
	MOVO  X3, X0
	MOVO  X6, X3
	MOVO  X9, X6
	JMP   openSSETail64DecLoop

openSSETail64DecLoopDone:
	MOVO X0, X1
	JMP  openSSETail16

openSSETail128:
	MOVO  ·chacha20Constants<>+0(SB), X1
	MOVO  32(BP), X4
	MOVO  48(BP), X7
	MOVO  128(BP), X10
	PADDL ·sseIncMask<>+0(SB), X10
	MOVO  X10, 80(BP)
	MOVO  X1, X0
	MOVO  X4, X3
	MOVO  X7, X6
	MOVO  X10, X9
	PADDL ·sseIncMask<>+0(SB), X9
	MOVO  X9, 96(BP)
	XORQ  R9, R9
	MOVQ  BX, CX
	ANDQ  $-16, CX

openSSETail128LoopA:
	ADDQ  (SI)(R9*1), R10
	ADCQ  8(SI)(R9*1), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12

openSSETail128LoopB:
	ADDQ  $0x10, R9
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X3
	PXOR  X12, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X3
	PXOR  X12, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X4
	PXOR  X12, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X4
	PXOR  X12, X4
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x0c
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X3
	PXOR  X12, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X3
	PXOR  X12, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X4
	PXOR  X12, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X4
	PXOR  X12, X4
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x04
	CMPQ  R9, CX
	JB    openSSETail128LoopA
	CMPQ  R9, $0xa0
	JNE   openSSETail128LoopB
	PADDL ·chacha20Constants<>+0(SB), X0
	PADDL ·chacha20Constants<>+0(SB), X1
	PADDL 32(BP), X3
	PADDL 32(BP), X4
	PADDL 48(BP), X6
	PADDL 48(BP), X7
	PADDL 96(BP), X9
	PADDL 80(BP), X10
	MOVOU (SI), X12
	MOVOU 16(SI), X13
	MOVOU 32(SI), X14
	MOVOU 48(SI), X15
	PXOR  X12, X1
	PXOR  X13, X4
	PXOR  X14, X7
	PXOR  X15, X10
	MOVOU X1, (DI)
	MOVOU X4, 16(DI)
	MOVOU X7, 32(DI)
	MOVOU X10, 48(DI)
	SUBQ  $0x40, BX
	LEAQ  64(SI), SI
	LEAQ  64(DI), DI
	JMP   openSSETail64DecLoop

openSSETail192:
	MOVO    ·chacha20Constants<>+0(SB), X2
	MOVO    32(BP), X5
	MOVO    48(BP), X8
	MOVO    128(BP), X11
	PADDL   ·sseIncMask<>+0(SB), X11
	MOVO    X11, 80(BP)
	MOVO    X2, X1
	MOVO    X5, X4
	MOVO    X8, X7
	MOVO    X11, X10
	PADDL   ·sseIncMask<>+0(SB), X10
	MOVO    X10, 96(BP)
	MOVO    X1, X0
	MOVO    X4, X3
	MOVO    X7, X6
	MOVO    X10, X9
	PADDL   ·sseIncMask<>+0(SB), X9
	MOVO    X9, 112(BP)
	MOVQ    BX, CX
	MOVQ    $0x000000a0, R9
	CMPQ    CX, $0xa0
	CMOVQGT R9, CX
	ANDQ    $-16, CX
	XORQ    R9, R9

openSSLTail192LoopA:
	ADDQ  (SI)(R9*1), R10
	ADCQ  8(SI)(R9*1), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12

openSSLTail192LoopB:
	ADDQ  $0x10, R9
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X3
	PXOR  X12, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X3
	PXOR  X12, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X4
	PXOR  X12, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X4
	PXOR  X12, X4
	PADDD X5, X2
	PXOR  X2, X11
	ROL16(X11, X12)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X5
	PXOR  X12, X5
	PADDD X5, X2
	PXOR  X2, X11
	ROL8(X11, X12)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X5
	PXOR  X12, X5
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc0
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X3
	PXOR  X12, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X3
	PXOR  X12, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X4
	PXOR  X12, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X4
	PXOR  X12, X4
	PADDD X5, X2
	PXOR  X2, X11
	ROL16(X11, X12)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X5
	PXOR  X12, X5
	PADDD X5, X2
	PXOR  X2, X11
	ROL8(X11, X12)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X5
	PXOR  X12, X5
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc0
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	CMPQ  R9, CX
	JB    openSSLTail192LoopA
	CMPQ  R9, $0xa0
	JNE   openSSLTail192LoopB
	CMPQ  BX, $0xb0
	JB    openSSLTail192Store
	ADDQ  160(SI), R10
	ADCQ  168(SI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	CMPQ  BX, $0xc0
	JB    openSSLTail192Store
	ADDQ  176(SI), R10
	ADCQ  184(SI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12

openSSLTail192Store:
	PADDL ·chacha20Constants<>+0(SB), X0
	PADDL ·chacha20Constants<>+0(SB), X1
	PADDL ·chacha20Constants<>+0(SB), X2
	PADDL 32(BP), X3
	PADDL 32(BP), X4
	PADDL 32(BP), X5
	PADDL 48(BP), X6
	PADDL 48(BP), X7
	PADDL 48(BP), X8
	PADDL 112(BP), X9
	PADDL 96(BP), X10
	PADDL 80(BP), X11
	MOVOU (SI), X12
	MOVOU 16(SI), X13
	MOVOU 32(SI), X14
	MOVOU 48(SI), X15
	PXOR  X12, X2
	PXOR  X13, X5
	PXOR  X14, X8
	PXOR  X15, X11
	MOVOU X2, (DI)
	MOVOU X5, 16(DI)
	MOVOU X8, 32(DI)
	MOVOU X11, 48(DI)
	MOVOU 64(SI), X12
	MOVOU 80(SI), X13
	MOVOU 96(SI), X14
	MOVOU 112(SI), X15
	PXOR  X12, X1
	PXOR  X13, X4
	PXOR  X14, X7
	PXOR  X15, X10
	MOVOU X1, 64(DI)
	MOVOU X4, 80(DI)
	MOVOU X7, 96(DI)
	MOVOU X10, 112(DI)
	SUBQ  $0x80, BX
	LEAQ  128(SI), SI
	LEAQ  128(DI), DI
	JMP   openSSETail64DecLoop

openSSETail256:
	MOVO  ·chacha20Constants<>+0(SB), X0
	MOVO  32(BP), X3
	MOVO  48(BP), X6
	MOVO  128(BP), X9
	PADDL ·sseIncMask<>+0(SB), X9
	MOVO  X0, X1
	MOVO  X3, X4
	MOVO  X6, X7
	MOVO  X9, X10
	PADDL ·sseIncMask<>+0(SB), X10
	MOVO  X1, X2
	MOVO  X4, X5
	MOVO  X7, X8
	MOVO  X10, X11
	PADDL ·sseIncMask<>+0(SB), X11
	MOVO  X2, X12
	MOVO  X5, X13
	MOVO  X8, X14
	MOVO  X11, X15
	PADDL ·sseIncMask<>+0(SB), X15

	// Store counters
	MOVO X9, 80(BP)
	MOVO X10, 96(BP)
	MOVO X11, 112(BP)
	MOVO X15, 128(BP)
	XORQ R9, R9

openSSETail256Loop:
	ADDQ  (SI)(R9*1), R10
	ADCQ  8(SI)(R9*1), R11
	ADCQ  $0x01, R12
	MOVO  X14, 64(BP)
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X14)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X3
	PXOR  X14, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X14)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X3
	PXOR  X14, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X14)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X4
	PXOR  X14, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X14)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X4
	PXOR  X14, X4
	PADDD X5, X2
	PXOR  X2, X11
	ROL16(X11, X14)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X5
	PXOR  X14, X5
	PADDD X5, X2
	PXOR  X2, X11
	ROL8(X11, X14)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X5
	PXOR  X14, X5
	MOVO  64(BP), X14
	MOVO  X7, 64(BP)
	PADDD X13, X12
	PXOR  X12, X15
	ROL16(X15, X7)
	PADDD X15, X14
	PXOR  X14, X13
	MOVO  X13, X7
	PSLLL $0x0c, X7
	PSRLL $0x14, X13
	PXOR  X7, X13
	PADDD X13, X12
	PXOR  X12, X15
	ROL8(X15, X7)
	PADDD X15, X14
	PXOR  X14, X13
	MOVO  X13, X7
	PSLLL $0x07, X7
	PSRLL $0x19, X13
	PXOR  X7, X13
	MOVO  64(BP), X7
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc0
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x0c
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	MOVO  X14, 64(BP)
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X14)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X3
	PXOR  X14, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X14)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X3
	PXOR  X14, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X14)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X4
	PXOR  X14, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X14)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X4
	PXOR  X14, X4
	PADDD X5, X2
	PXOR  X2, X11
	ROL16(X11, X14)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X5
	PXOR  X14, X5
	PADDD X5, X2
	PXOR  X2, X11
	ROL8(X11, X14)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X5
	PXOR  X14, X5
	MOVO  64(BP), X14
	MOVO  X7, 64(BP)
	PADDD X13, X12
	PXOR  X12, X15
	ROL16(X15, X7)
	PADDD X15, X14
	PXOR  X14, X13
	MOVO  X13, X7
	PSLLL $0x0c, X7
	PSRLL $0x14, X13
	PXOR  X7, X13
	PADDD X13, X12
	PXOR  X12, X15
	ROL8(X15, X7)
	PADDD X15, X14
	PXOR  X14, X13
	MOVO  X13, X7
	PSLLL $0x07, X7
	PSRLL $0x19, X13
	PXOR  X7, X13
	MOVO  64(BP), X7
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc0
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x04
	ADDQ  $0x10, R9
	CMPQ  R9, $0xa0
	JB    openSSETail256Loop
	MOVQ  BX, CX
	ANDQ  $-16, CX

openSSETail256HashLoop:
	ADDQ  (SI)(R9*1), R10
	ADCQ  8(SI)(R9*1), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	ADDQ  $0x10, R9
	CMPQ  R9, CX
	JB    openSSETail256HashLoop

	// Add in the state
	PADDD ·chacha20Constants<>+0(SB), X0
	PADDD ·chacha20Constants<>+0(SB), X1
	PADDD ·chacha20Constants<>+0(SB), X2
	PADDD ·chacha20Constants<>+0(SB), X12
	PADDD 32(BP), X3
	PADDD 32(BP), X4
	PADDD 32(BP), X5
	PADDD 32(BP), X13
	PADDD 48(BP), X6
	PADDD 48(BP), X7
	PADDD 48(BP), X8
	PADDD 48(BP), X14
	PADDD 80(BP), X9
	PADDD 96(BP), X10
	PADDD 112(BP), X11
	PADDD 128(BP), X15
	MOVO  X15, 64(BP)

	// Load - xor - store
	MOVOU (SI), X15
	PXOR  X15, X0
	MOVOU 16(SI), X15
	PXOR  X15, X3
	MOVOU 32(SI), X15
	PXOR  X15, X6
	MOVOU 48(SI), X15
	PXOR  X15, X9
	MOVOU X0, (DI)
	MOVOU X3, 16(DI)
	MOVOU X6, 32(DI)
	MOVOU X9, 48(DI)
	MOVOU 64(SI), X0
	MOVOU 80(SI), X3
	MOVOU 96(SI), X6
	MOVOU 112(SI), X9
	PXOR  X0, X1
	PXOR  X3, X4
	PXOR  X6, X7
	PXOR  X9, X10
	MOVOU X1, 64(DI)
	MOVOU X4, 80(DI)
	MOVOU X7, 96(DI)
	MOVOU X10, 112(DI)
	MOVOU 128(SI), X0
	MOVOU 144(SI), X3
	MOVOU 160(SI), X6
	MOVOU 176(SI), X9
	PXOR  X0, X2
	PXOR  X3, X5
	PXOR  X6, X8
	PXOR  X9, X11
	MOVOU X2, 128(DI)
	MOVOU X5, 144(DI)
	MOVOU X8, 160(DI)
	MOVOU X11, 176(DI)
	LEAQ  192(SI), SI
	LEAQ  192(DI), DI
	SUBQ  $0xc0, BX
	MOVO  X12, X0
	MOVO  X13, X3
	MOVO  X14, X6
	MOVO  64(BP), X9
	JMP   openSSETail64DecLoop

chacha20Poly1305Open_AVX2:
	VZEROUPPER
	VMOVDQU ·chacha20Constants<>+0(SB), Y0
	BYTE    $0xc4
	BYTE    $0x42
	BYTE    $0x7d
	BYTE    $0x5a
	BYTE    $0x70
	BYTE    $0x10
	BYTE    $0xc4
	BYTE    $0x42
	BYTE    $0x7d
	BYTE    $0x5a
	BYTE    $0x60
	BYTE    $0x20
	BYTE    $0xc4
	BYTE    $0xc2
	BYTE    $0x7d
	BYTE    $0x5a
	BYTE    $0x60
	BYTE    $0x30
	VPADDD  ·avx2InitMask<>+0(SB), Y4, Y4

	// Special optimization, for very short buffers
	CMPQ BX, $0xc0
	JBE  openAVX2192
	CMPQ BX, $0x00000140
	JBE  openAVX2320

	// For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream
	VMOVDQA Y14, 32(BP)
	VMOVDQA Y12, 64(BP)
	VMOVDQA Y4, 192(BP)
	MOVQ    $0x0000000a, R9

openAVX2PreparePolyKey:
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x0c, Y14, Y3
	VPSRLD     $0x14, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x07, Y14, Y3
	VPSRLD     $0x19, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPALIGNR   $0x04, Y14, Y14, Y14
	VPALIGNR   $0x08, Y12, Y12, Y12
	VPALIGNR   $0x0c, Y4, Y4, Y4
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x0c, Y14, Y3
	VPSRLD     $0x14, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x07, Y14, Y3
	VPSRLD     $0x19, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPALIGNR   $0x0c, Y14, Y14, Y14
	VPALIGNR   $0x08, Y12, Y12, Y12
	VPALIGNR   $0x04, Y4, Y4, Y4
	DECQ       R9
	JNE        openAVX2PreparePolyKey
	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
	VPADDD     32(BP), Y14, Y14
	VPADDD     64(BP), Y12, Y12
	VPADDD     192(BP), Y4, Y4
	VPERM2I128 $0x02, Y0, Y14, Y3

	// Clamp and store poly key
	VPAND   ·polyClampMask<>+0(SB), Y3, Y3
	VMOVDQA Y3, (BP)

	// Stream for the first 64 bytes
	VPERM2I128 $0x13, Y0, Y14, Y0
	VPERM2I128 $0x13, Y12, Y4, Y14

	// Hash AD + first 64 bytes
	MOVQ ad_len+80(FP), R9
	CALL polyHashADInternal<>(SB)
	XORQ CX, CX

openAVX2InitialHash64:
	ADDQ  (SI)(CX*1), R10
	ADCQ  8(SI)(CX*1), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), DX
	MOVQ  DX, R15
	MULXQ R10, R13, R14
	IMULQ R12, R15
	MULXQ R11, AX, DX
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), DX
	MULXQ R10, R10, AX
	ADDQ  R10, R14
	MULXQ R11, R11, R8
	ADCQ  R11, R15
	ADCQ  $0x00, R8
	IMULQ R12, DX
	ADDQ  AX, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	ADDQ  $0x10, CX
	CMPQ  CX, $0x40
	JNE   openAVX2InitialHash64

	// Decrypt the first 64 bytes
	VPXOR   (SI), Y0, Y0
	VPXOR   32(SI), Y14, Y14
	VMOVDQU Y0, (DI)
	VMOVDQU Y14, 32(DI)
	LEAQ    64(SI), SI
	LEAQ    64(DI), DI
	SUBQ    $0x40, BX

openAVX2MainLoop:
	CMPQ BX, $0x00000200
	JB   openAVX2MainLoopDone

	// Load state, increment counter blocks, store the incremented counters
	VMOVDQU ·chacha20Constants<>+0(SB), Y0
	VMOVDQA Y0, Y5
	VMOVDQA Y0, Y6
	VMOVDQA Y0, Y7
	VMOVDQA 32(BP), Y14
	VMOVDQA Y14, Y9
	VMOVDQA Y14, Y10
	VMOVDQA Y14, Y11
	VMOVDQA 64(BP), Y12
	VMOVDQA Y12, Y13
	VMOVDQA Y12, Y8
	VMOVDQA Y12, Y15
	VMOVDQA 192(BP), Y4
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
	VMOVDQA Y4, 96(BP)
	VMOVDQA Y1, 128(BP)
	VMOVDQA Y2, 160(BP)
	VMOVDQA Y3, 192(BP)
	XORQ    CX, CX

openAVX2InternalLoop:
	ADDQ     (SI)(CX*1), R10
	ADCQ     8(SI)(CX*1), R11
	ADCQ     $0x01, R12
	VPADDD   Y14, Y0, Y0
	VPADDD   Y9, Y5, Y5
	VPADDD   Y10, Y6, Y6
	VPADDD   Y11, Y7, Y7
	MOVQ     (BP), DX
	MOVQ     DX, R15
	MULXQ    R10, R13, R14
	IMULQ    R12, R15
	MULXQ    R11, AX, DX
	ADDQ     AX, R14
	ADCQ     DX, R15
	VPXOR    Y0, Y4, Y4
	VPXOR    Y5, Y1, Y1
	VPXOR    Y6, Y2, Y2
	VPXOR    Y7, Y3, Y3
	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
	MOVQ     8(BP), DX
	MULXQ    R10, R10, AX
	ADDQ     R10, R14
	MULXQ    R11, R11, R8
	ADCQ     R11, R15
	ADCQ     $0x00, R8
	VPADDD   Y4, Y12, Y12
	VPADDD   Y1, Y13, Y13
	VPADDD   Y2, Y8, Y8
	VPADDD   Y3, Y15, Y15
	VPXOR    Y12, Y14, Y14
	VPXOR    Y13, Y9, Y9
	VPXOR    Y8, Y10, Y10
	VPXOR    Y15, Y11, Y11
	IMULQ    R12, DX
	ADDQ     AX, R15
	ADCQ     DX, R8
	VMOVDQA  Y15, 224(BP)
	VPSLLD   $0x0c, Y14, Y15
	VPSRLD   $0x14, Y14, Y14
	VPXOR    Y15, Y14, Y14
	VPSLLD   $0x0c, Y9, Y15
	VPSRLD   $0x14, Y9, Y9
	VPXOR    Y15, Y9, Y9
	VPSLLD   $0x0c, Y10, Y15
	VPSRLD   $0x14, Y10, Y10
	VPXOR    Y15, Y10, Y10
	VPSLLD   $0x0c, Y11, Y15
	VPSRLD   $0x14, Y11, Y11
	VPXOR    Y15, Y11, Y11
	VMOVDQA  224(BP), Y15
	MOVQ     R13, R10
	MOVQ     R14, R11
	MOVQ     R15, R12
	ANDQ     $0x03, R12
	MOVQ     R15, R13
	ANDQ     $-4, R13
	MOVQ     R8, R14
	SHRQ     $0x02, R8, R15
	SHRQ     $0x02, R8
	ADDQ     R13, R10
	ADCQ     R14, R11
	ADCQ     $0x00, R12
	ADDQ     R15, R10
	ADCQ     R8, R11
	ADCQ     $0x00, R12
	VPADDD   Y14, Y0, Y0
	VPADDD   Y9, Y5, Y5
	VPADDD   Y10, Y6, Y6
	VPADDD   Y11, Y7, Y7
	VPXOR    Y0, Y4, Y4
	VPXOR    Y5, Y1, Y1
	VPXOR    Y6, Y2, Y2
	VPXOR    Y7, Y3, Y3
	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
	ADDQ     16(SI)(CX*1), R10
	ADCQ     24(SI)(CX*1), R11
	ADCQ     $0x01, R12
	VPADDD   Y4, Y12, Y12
	VPADDD   Y1, Y13, Y13
	VPADDD   Y2, Y8, Y8
	VPADDD   Y3, Y15, Y15
	MOVQ     (BP), DX
	MOVQ     DX, R15
	MULXQ    R10, R13, R14
	IMULQ    R12, R15
	MULXQ    R11, AX, DX
	ADDQ     AX, R14
	ADCQ     DX, R15
	VPXOR    Y12, Y14, Y14
	VPXOR    Y13, Y9, Y9
	VPXOR    Y8, Y10, Y10
	VPXOR    Y15, Y11, Y11
	VMOVDQA  Y15, 224(BP)
	VPSLLD   $0x07, Y14, Y15
	VPSRLD   $0x19, Y14, Y14
	VPXOR    Y15, Y14, Y14
	VPSLLD   $0x07, Y9, Y15
	VPSRLD   $0x19, Y9, Y9
	VPXOR    Y15, Y9, Y9
	VPSLLD   $0x07, Y10, Y15
	VPSRLD   $0x19, Y10, Y10
	VPXOR    Y15, Y10, Y10
	VPSLLD   $0x07, Y11, Y15
	VPSRLD   $0x19, Y11, Y11
	VPXOR    Y15, Y11, Y11
	VMOVDQA  224(BP), Y15
	MOVQ     8(BP), DX
	MULXQ    R10, R10, AX
	ADDQ     R10, R14
	MULXQ    R11, R11, R8
	ADCQ     R11, R15
	ADCQ     $0x00, R8
	VPALIGNR $0x04, Y14, Y14, Y14
	VPALIGNR $0x04, Y9, Y9, Y9
	VPALIGNR $0x04, Y10, Y10, Y10
	VPALIGNR $0x04, Y11, Y11, Y11
	VPALIGNR $0x08, Y12, Y12, Y12
	VPALIGNR $0x08, Y13, Y13, Y13
	VPALIGNR $0x08, Y8, Y8, Y8
	VPALIGNR $0x08, Y15, Y15, Y15
	VPALIGNR $0x0c, Y4, Y4, Y4
	VPALIGNR $0x0c, Y1, Y1, Y1
	VPALIGNR $0x0c, Y2, Y2, Y2
	VPALIGNR $0x0c, Y3, Y3, Y3
	VPADDD   Y14, Y0, Y0
	VPADDD   Y9, Y5, Y5
	VPADDD   Y10, Y6, Y6
	VPADDD   Y11, Y7, Y7
	IMULQ    R12, DX
	ADDQ     AX, R15
	ADCQ     DX, R8
	VPXOR    Y0, Y4, Y4
	VPXOR    Y5, Y1, Y1
	VPXOR    Y6, Y2, Y2
	VPXOR    Y7, Y3, Y3
	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
	MOVQ     R13, R10
	MOVQ     R14, R11
	MOVQ     R15, R12
	ANDQ     $0x03, R12
	MOVQ     R15, R13
	ANDQ     $-4, R13
	MOVQ     R8, R14
	SHRQ     $0x02, R8, R15
	SHRQ     $0x02, R8
	ADDQ     R13, R10
	ADCQ     R14, R11
	ADCQ     $0x00, R12
	ADDQ     R15, R10
	ADCQ     R8, R11
	ADCQ     $0x00, R12
	VPADDD   Y4, Y12, Y12
	VPADDD   Y1, Y13, Y13
	VPADDD   Y2, Y8, Y8
	VPADDD   Y3, Y15, Y15
	VPXOR    Y12, Y14, Y14
	VPXOR    Y13, Y9, Y9
	VPXOR    Y8, Y10, Y10
	VPXOR    Y15, Y11, Y11
	ADDQ     32(SI)(CX*1), R10
	ADCQ     40(SI)(CX*1), R11
	ADCQ     $0x01, R12
	LEAQ     48(CX), CX
	VMOVDQA  Y15, 224(BP)
	VPSLLD   $0x0c, Y14, Y15
	VPSRLD   $0x14, Y14, Y14
	VPXOR    Y15, Y14, Y14
	VPSLLD   $0x0c, Y9, Y15
	VPSRLD   $0x14, Y9, Y9
	VPXOR    Y15, Y9, Y9
	VPSLLD   $0x0c, Y10, Y15
	VPSRLD   $0x14, Y10, Y10
	VPXOR    Y15, Y10, Y10
	VPSLLD   $0x0c, Y11, Y15
	VPSRLD   $0x14, Y11, Y11
	VPXOR    Y15, Y11, Y11
	VMOVDQA  224(BP), Y15
	MOVQ     (BP), DX
	MOVQ     DX, R15
	MULXQ    R10, R13, R14
	IMULQ    R12, R15
	MULXQ    R11, AX, DX
	ADDQ     AX, R14
	ADCQ     DX, R15
	VPADDD   Y14, Y0, Y0
	VPADDD   Y9, Y5, Y5
	VPADDD   Y10, Y6, Y6
	VPADDD   Y11, Y7, Y7
	VPXOR    Y0, Y4, Y4
	VPXOR    Y5, Y1, Y1
	VPXOR    Y6, Y2, Y2
	VPXOR    Y7, Y3, Y3
	MOVQ     8(BP), DX
	MULXQ    R10, R10, AX
	ADDQ     R10, R14
	MULXQ    R11, R11, R8
	ADCQ     R11, R15
	ADCQ     $0x00, R8
	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
	VPADDD   Y4, Y12, Y12
	VPADDD   Y1, Y13, Y13
	VPADDD   Y2, Y8, Y8
	VPADDD   Y3, Y15, Y15
	IMULQ    R12, DX
	ADDQ     AX, R15
	ADCQ     DX, R8
	VPXOR    Y12, Y14, Y14
	VPXOR    Y13, Y9, Y9
	VPXOR    Y8, Y10, Y10
	VPXOR    Y15, Y11, Y11
	VMOVDQA  Y15, 224(BP)
	VPSLLD   $0x07, Y14, Y15
	VPSRLD   $0x19, Y14, Y14
	VPXOR    Y15, Y14, Y14
	VPSLLD   $0x07, Y9, Y15
	VPSRLD   $0x19, Y9, Y9
	VPXOR    Y15, Y9, Y9
	VPSLLD   $0x07, Y10, Y15
	VPSRLD   $0x19, Y10, Y10
	VPXOR    Y15, Y10, Y10
	VPSLLD   $0x07, Y11, Y15
	VPSRLD   $0x19, Y11, Y11
	VPXOR    Y15, Y11, Y11
	VMOVDQA  224(BP), Y15
	MOVQ     R13, R10
	MOVQ     R14, R11
	MOVQ     R15, R12
	ANDQ     $0x03, R12
	MOVQ     R15, R13
	ANDQ     $-4, R13
	MOVQ     R8, R14
	SHRQ     $0x02, R8, R15
	SHRQ     $0x02, R8
	ADDQ     R13, R10
	ADCQ     R14, R11
	ADCQ     $0x00, R12
	ADDQ     R15, R10
	ADCQ     R8, R11
	ADCQ     $0x00, R12
	VPALIGNR $0x0c, Y14, Y14, Y14
	VPALIGNR $0x0c, Y9, Y9, Y9
	VPALIGNR $0x0c, Y10, Y10, Y10
	VPALIGNR $0x0c, Y11, Y11, Y11
	VPALIGNR $0x08, Y12, Y12, Y12
	VPALIGNR $0x08, Y13, Y13, Y13
	VPALIGNR $0x08, Y8, Y8, Y8
	VPALIGNR $0x08, Y15, Y15, Y15
	VPALIGNR $0x04, Y4, Y4, Y4
	VPALIGNR $0x04, Y1, Y1, Y1
	VPALIGNR $0x04, Y2, Y2, Y2
	VPALIGNR $0x04, Y3, Y3, Y3
	CMPQ     CX, $0x000001e0
	JNE      openAVX2InternalLoop
	VPADDD   ·chacha20Constants<>+0(SB), Y0, Y0
	VPADDD   ·chacha20Constants<>+0(SB), Y5, Y5
	VPADDD   ·chacha20Constants<>+0(SB), Y6, Y6
	VPADDD   ·chacha20Constants<>+0(SB), Y7, Y7
	VPADDD   32(BP), Y14, Y14
	VPADDD   32(BP), Y9, Y9
	VPADDD   32(BP), Y10, Y10
	VPADDD   32(BP), Y11, Y11
	VPADDD   64(BP), Y12, Y12
	VPADDD   64(BP), Y13, Y13
	VPADDD   64(BP), Y8, Y8
	VPADDD   64(BP), Y15, Y15
	VPADDD   96(BP), Y4, Y4
	VPADDD   128(BP), Y1, Y1
	VPADDD   160(BP), Y2, Y2
	VPADDD   192(BP), Y3, Y3
	VMOVDQA  Y15, 224(BP)

	// We only hashed 480 of the 512 bytes available - hash the remaining 32 here
	ADDQ       480(SI), R10
	ADCQ       488(SI), R11
	ADCQ       $0x01, R12
	MOVQ       (BP), DX
	MOVQ       DX, R15
	MULXQ      R10, R13, R14
	IMULQ      R12, R15
	MULXQ      R11, AX, DX
	ADDQ       AX, R14
	ADCQ       DX, R15
	MOVQ       8(BP), DX
	MULXQ      R10, R10, AX
	ADDQ       R10, R14
	MULXQ      R11, R11, R8
	ADCQ       R11, R15
	ADCQ       $0x00, R8
	IMULQ      R12, DX
	ADDQ       AX, R15
	ADCQ       DX, R8
	MOVQ       R13, R10
	MOVQ       R14, R11
	MOVQ       R15, R12
	ANDQ       $0x03, R12
	MOVQ       R15, R13
	ANDQ       $-4, R13
	MOVQ       R8, R14
	SHRQ       $0x02, R8, R15
	SHRQ       $0x02, R8
	ADDQ       R13, R10
	ADCQ       R14, R11
	ADCQ       $0x00, R12
	ADDQ       R15, R10
	ADCQ       R8, R11
	ADCQ       $0x00, R12
	VPERM2I128 $0x02, Y0, Y14, Y15
	VPERM2I128 $0x13, Y0, Y14, Y14
	VPERM2I128 $0x02, Y12, Y4, Y0
	VPERM2I128 $0x13, Y12, Y4, Y12
	VPXOR      (SI), Y15, Y15
	VPXOR      32(SI), Y0, Y0
	VPXOR      64(SI), Y14, Y14
	VPXOR      96(SI), Y12, Y12
	VMOVDQU    Y15, (DI)
	VMOVDQU    Y0, 32(DI)
	VMOVDQU    Y14, 64(DI)
	VMOVDQU    Y12, 96(DI)
	VPERM2I128 $0x02, Y5, Y9, Y0
	VPERM2I128 $0x02, Y13, Y1, Y14
	VPERM2I128 $0x13, Y5, Y9, Y12
	VPERM2I128 $0x13, Y13, Y1, Y4
	VPXOR      128(SI), Y0, Y0
	VPXOR      160(SI), Y14, Y14
	VPXOR      192(SI), Y12, Y12
	VPXOR      224(SI), Y4, Y4
	VMOVDQU    Y0, 128(DI)
	VMOVDQU    Y14, 160(DI)
	VMOVDQU    Y12, 192(DI)
	VMOVDQU    Y4, 224(DI)

	// and here
	ADDQ       496(SI), R10
	ADCQ       504(SI), R11
	ADCQ       $0x01, R12
	MOVQ       (BP), DX
	MOVQ       DX, R15
	MULXQ      R10, R13, R14
	IMULQ      R12, R15
	MULXQ      R11, AX, DX
	ADDQ       AX, R14
	ADCQ       DX, R15
	MOVQ       8(BP), DX
	MULXQ      R10, R10, AX
	ADDQ       R10, R14
	MULXQ      R11, R11, R8
	ADCQ       R11, R15
	ADCQ       $0x00, R8
	IMULQ      R12, DX
	ADDQ       AX, R15
	ADCQ       DX, R8
	MOVQ       R13, R10
	MOVQ       R14, R11
	MOVQ       R15, R12
	ANDQ       $0x03, R12
	MOVQ       R15, R13
	ANDQ       $-4, R13
	MOVQ       R8, R14
	SHRQ       $0x02, R8, R15
	SHRQ       $0x02, R8
	ADDQ       R13, R10
	ADCQ       R14, R11
	ADCQ       $0x00, R12
	ADDQ       R15, R10
	ADCQ       R8, R11
	ADCQ       $0x00, R12
	VPERM2I128 $0x02, Y6, Y10, Y0
	VPERM2I128 $0x02, Y8, Y2, Y14
	VPERM2I128 $0x13, Y6, Y10, Y12
	VPERM2I128 $0x13, Y8, Y2, Y4
	VPXOR      256(SI), Y0, Y0
	VPXOR      288(SI), Y14, Y14
	VPXOR      320(SI), Y12, Y12
	VPXOR      352(SI), Y4, Y4
	VMOVDQU    Y0, 256(DI)
	VMOVDQU    Y14, 288(DI)
	VMOVDQU    Y12, 320(DI)
	VMOVDQU    Y4, 352(DI)
	VPERM2I128 $0x02, Y7, Y11, Y0
	VPERM2I128 $0x02, 224(BP), Y3, Y14
	VPERM2I128 $0x13, Y7, Y11, Y12
	VPERM2I128 $0x13, 224(BP), Y3, Y4
	VPXOR      384(SI), Y0, Y0
	VPXOR      416(SI), Y14, Y14
	VPXOR      448(SI), Y12, Y12
	VPXOR      480(SI), Y4, Y4
	VMOVDQU    Y0, 384(DI)
	VMOVDQU    Y14, 416(DI)
	VMOVDQU    Y12, 448(DI)
	VMOVDQU    Y4, 480(DI)
	LEAQ       512(SI), SI
	LEAQ       512(DI), DI
	SUBQ       $0x00000200, BX
	JMP        openAVX2MainLoop

openAVX2MainLoopDone:
	// Handle the various tail sizes efficiently
	TESTQ BX, BX
	JE    openSSEFinalize
	CMPQ  BX, $0x80
	JBE   openAVX2Tail128
	CMPQ  BX, $0x00000100
	JBE   openAVX2Tail256
	CMPQ  BX, $0x00000180
	JBE   openAVX2Tail384
	JMP   openAVX2Tail512

openAVX2192:
	VMOVDQA Y0, Y5
	VMOVDQA Y14, Y9
	VMOVDQA Y12, Y13
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
	VMOVDQA Y0, Y6
	VMOVDQA Y14, Y10
	VMOVDQA Y12, Y8
	VMOVDQA Y4, Y2
	VMOVDQA Y1, Y15
	MOVQ    $0x0000000a, R9

openAVX2192InnerCipherLoop:
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x0c, Y14, Y3
	VPSRLD     $0x14, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x07, Y14, Y3
	VPSRLD     $0x19, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x0c, Y9, Y3
	VPSRLD     $0x14, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x07, Y9, Y3
	VPSRLD     $0x19, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPALIGNR   $0x04, Y14, Y14, Y14
	VPALIGNR   $0x04, Y9, Y9, Y9
	VPALIGNR   $0x08, Y12, Y12, Y12
	VPALIGNR   $0x08, Y13, Y13, Y13
	VPALIGNR   $0x0c, Y4, Y4, Y4
	VPALIGNR   $0x0c, Y1, Y1, Y1
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x0c, Y14, Y3
	VPSRLD     $0x14, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x07, Y14, Y3
	VPSRLD     $0x19, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x0c, Y9, Y3
	VPSRLD     $0x14, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x07, Y9, Y3
	VPSRLD     $0x19, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPALIGNR   $0x0c, Y14, Y14, Y14
	VPALIGNR   $0x0c, Y9, Y9, Y9
	VPALIGNR   $0x08, Y12, Y12, Y12
	VPALIGNR   $0x08, Y13, Y13, Y13
	VPALIGNR   $0x04, Y4, Y4, Y4
	VPALIGNR   $0x04, Y1, Y1, Y1
	DECQ       R9
	JNE        openAVX2192InnerCipherLoop
	VPADDD     Y6, Y0, Y0
	VPADDD     Y6, Y5, Y5
	VPADDD     Y10, Y14, Y14
	VPADDD     Y10, Y9, Y9
	VPADDD     Y8, Y12, Y12
	VPADDD     Y8, Y13, Y13
	VPADDD     Y2, Y4, Y4
	VPADDD     Y15, Y1, Y1
	VPERM2I128 $0x02, Y0, Y14, Y3

	// Clamp and store poly key
	VPAND   ·polyClampMask<>+0(SB), Y3, Y3
	VMOVDQA Y3, (BP)

	// Stream for up to 192 bytes
	VPERM2I128 $0x13, Y0, Y14, Y0
	VPERM2I128 $0x13, Y12, Y4, Y14
	VPERM2I128 $0x02, Y5, Y9, Y12
	VPERM2I128 $0x02, Y13, Y1, Y4
	VPERM2I128 $0x13, Y5, Y9, Y5
	VPERM2I128 $0x13, Y13, Y1, Y9

openAVX2ShortOpen:
	// Hash
	MOVQ ad_len+80(FP), R9
	CALL polyHashADInternal<>(SB)

openAVX2ShortOpenLoop:
	CMPQ BX, $0x20
	JB   openAVX2ShortTail32
	SUBQ $0x20, BX

	// Load for hashing
	ADDQ  (SI), R10
	ADCQ  8(SI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), DX
	MOVQ  DX, R15
	MULXQ R10, R13, R14
	IMULQ R12, R15
	MULXQ R11, AX, DX
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), DX
	MULXQ R10, R10, AX
	ADDQ  R10, R14
	MULXQ R11, R11, R8
	ADCQ  R11, R15
	ADCQ  $0x00, R8
	IMULQ R12, DX
	ADDQ  AX, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	ADDQ  16(SI), R10
	ADCQ  24(SI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), DX
	MOVQ  DX, R15
	MULXQ R10, R13, R14
	IMULQ R12, R15
	MULXQ R11, AX, DX
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), DX
	MULXQ R10, R10, AX
	ADDQ  R10, R14
	MULXQ R11, R11, R8
	ADCQ  R11, R15
	ADCQ  $0x00, R8
	IMULQ R12, DX
	ADDQ  AX, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12

	// Load for decryption
	VPXOR   (SI), Y0, Y0
	VMOVDQU Y0, (DI)
	LEAQ    32(SI), SI
	LEAQ    32(DI), DI

	// Shift stream left
	VMOVDQA Y14, Y0
	VMOVDQA Y12, Y14
	VMOVDQA Y4, Y12
	VMOVDQA Y5, Y4
	VMOVDQA Y9, Y5
	VMOVDQA Y13, Y9
	VMOVDQA Y1, Y13
	VMOVDQA Y6, Y1
	VMOVDQA Y10, Y6
	JMP     openAVX2ShortOpenLoop

openAVX2ShortTail32:
	CMPQ    BX, $0x10
	VMOVDQA X0, X1
	JB      openAVX2ShortDone
	SUBQ    $0x10, BX

	// Load for hashing
	ADDQ  (SI), R10
	ADCQ  8(SI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), DX
	MOVQ  DX, R15
	MULXQ R10, R13, R14
	IMULQ R12, R15
	MULXQ R11, AX, DX
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), DX
	MULXQ R10, R10, AX
	ADDQ  R10, R14
	MULXQ R11, R11, R8
	ADCQ  R11, R15
	ADCQ  $0x00, R8
	IMULQ R12, DX
	ADDQ  AX, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12

	// Load for decryption
	VPXOR      (SI), X0, X12
	VMOVDQU    X12, (DI)
	LEAQ       16(SI), SI
	LEAQ       16(DI), DI
	VPERM2I128 $0x11, Y0, Y0, Y0
	VMOVDQA    X0, X1

openAVX2ShortDone:
	VZEROUPPER
	JMP openSSETail16

openAVX2320:
	VMOVDQA Y0, Y5
	VMOVDQA Y14, Y9
	VMOVDQA Y12, Y13
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
	VMOVDQA Y0, Y6
	VMOVDQA Y14, Y10
	VMOVDQA Y12, Y8
	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
	VMOVDQA Y14, Y7
	VMOVDQA Y12, Y11
	VMOVDQA Y4, Y15
	MOVQ    $0x0000000a, R9

openAVX2320InnerCipherLoop:
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x0c, Y14, Y3
	VPSRLD   $0x14, Y14, Y14
	VPXOR    Y3, Y14, Y14
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x07, Y14, Y3
	VPSRLD   $0x19, Y14, Y14
	VPXOR    Y3, Y14, Y14
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x0c, Y9, Y3
	VPSRLD   $0x14, Y9, Y9
	VPXOR    Y3, Y9, Y9
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x07, Y9, Y3
	VPSRLD   $0x19, Y9, Y9
	VPXOR    Y3, Y9, Y9
	VPADDD   Y10, Y6, Y6
	VPXOR    Y6, Y2, Y2
	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
	VPADDD   Y2, Y8, Y8
	VPXOR    Y8, Y10, Y10
	VPSLLD   $0x0c, Y10, Y3
	VPSRLD   $0x14, Y10, Y10
	VPXOR    Y3, Y10, Y10
	VPADDD   Y10, Y6, Y6
	VPXOR    Y6, Y2, Y2
	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
	VPADDD   Y2, Y8, Y8
	VPXOR    Y8, Y10, Y10
	VPSLLD   $0x07, Y10, Y3
	VPSRLD   $0x19, Y10, Y10
	VPXOR    Y3, Y10, Y10
	VPALIGNR $0x04, Y14, Y14, Y14
	VPALIGNR $0x04, Y9, Y9, Y9
	VPALIGNR $0x04, Y10, Y10, Y10
	VPALIGNR $0x08, Y12, Y12, Y12
	VPALIGNR $0x08, Y13, Y13, Y13
	VPALIGNR $0x08, Y8, Y8, Y8
	VPALIGNR $0x0c, Y4, Y4, Y4
	VPALIGNR $0x0c, Y1, Y1, Y1
	VPALIGNR $0x0c, Y2, Y2, Y2
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x0c, Y14, Y3
	VPSRLD   $0x14, Y14, Y14
	VPXOR    Y3, Y14, Y14
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x07, Y14, Y3
	VPSRLD   $0x19, Y14, Y14
	VPXOR    Y3, Y14, Y14
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x0c, Y9, Y3
	VPSRLD   $0x14, Y9, Y9
	VPXOR    Y3, Y9, Y9
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x07, Y9, Y3
	VPSRLD   $0x19, Y9, Y9
	VPXOR    Y3, Y9, Y9
	VPADDD   Y10, Y6, Y6
	VPXOR    Y6, Y2, Y2
	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
	VPADDD   Y2, Y8, Y8
	VPXOR    Y8, Y10, Y10
	VPSLLD   $0x0c, Y10, Y3
	VPSRLD   $0x14, Y10, Y10
	VPXOR    Y3, Y10, Y10
	VPADDD   Y10, Y6, Y6
	VPXOR    Y6, Y2, Y2
	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
	VPADDD   Y2, Y8, Y8
	VPXOR    Y8, Y10, Y10
	VPSLLD   $0x07, Y10, Y3
	VPSRLD   $0x19, Y10, Y10
	VPXOR    Y3, Y10, Y10
	VPALIGNR $0x0c, Y14, Y14, Y14
	VPALIGNR $0x0c, Y9, Y9, Y9
	VPALIGNR $0x0c, Y10, Y10, Y10
	VPALIGNR $0x08, Y12, Y12, Y12
	VPALIGNR $0x08, Y13, Y13, Y13
	VPALIGNR $0x08, Y8, Y8, Y8
	VPALIGNR $0x04, Y4, Y4, Y4
	VPALIGNR $0x04, Y1, Y1, Y1
	VPALIGNR $0x04, Y2, Y2, Y2
	DECQ     R9
	JNE      openAVX2320InnerCipherLoop
	VMOVDQA  ·chacha20Constants<>+0(SB), Y3
	VPADDD   Y3, Y0, Y0
	VPADDD   Y3, Y5, Y5
	VPADDD   Y3, Y6, Y6
	VPADDD   Y7, Y14, Y14
	VPADDD   Y7, Y9, Y9
	VPADDD   Y7, Y10, Y10
	VPADDD   Y11, Y12, Y12
	VPADDD   Y11, Y13, Y13
	VPADDD   Y11, Y8, Y8
	VMOVDQA  ·avx2IncMask<>+0(SB), Y3
	VPADDD   Y15, Y4, Y4
	VPADDD   Y3, Y15, Y15
	VPADDD   Y15, Y1, Y1
	VPADDD   Y3, Y15, Y15
	VPADDD   Y15, Y2, Y2

	// Clamp and store poly key
	VPERM2I128 $0x02, Y0, Y14, Y3
	VPAND      ·polyClampMask<>+0(SB), Y3, Y3
	VMOVDQA    Y3, (BP)

	// Stream for up to 320 bytes
	VPERM2I128 $0x13, Y0, Y14, Y0
	VPERM2I128 $0x13, Y12, Y4, Y14
	VPERM2I128 $0x02, Y5, Y9, Y12
	VPERM2I128 $0x02, Y13, Y1, Y4
	VPERM2I128 $0x13, Y5, Y9, Y5
	VPERM2I128 $0x13, Y13, Y1, Y9
	VPERM2I128 $0x02, Y6, Y10, Y13
	VPERM2I128 $0x02, Y8, Y2, Y1
	VPERM2I128 $0x13, Y6, Y10, Y6
	VPERM2I128 $0x13, Y8, Y2, Y10
	JMP        openAVX2ShortOpen

openAVX2Tail128:
	// Need to decrypt up to 128 bytes - prepare two blocks
	VMOVDQA ·chacha20Constants<>+0(SB), Y5
	VMOVDQA 32(BP), Y9
	VMOVDQA 64(BP), Y13
	VMOVDQA 192(BP), Y1
	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y1
	VMOVDQA Y1, Y4
	XORQ    R9, R9
	MOVQ    BX, CX
	ANDQ    $-16, CX
	TESTQ   CX, CX
	JE      openAVX2Tail128LoopB

openAVX2Tail128LoopA:
	ADDQ  (SI)(R9*1), R10
	ADCQ  8(SI)(R9*1), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), DX
	MOVQ  DX, R15
	MULXQ R10, R13, R14
	IMULQ R12, R15
	MULXQ R11, AX, DX
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), DX
	MULXQ R10, R10, AX
	ADDQ  R10, R14
	MULXQ R11, R11, R8
	ADCQ  R11, R15
	ADCQ  $0x00, R8
	IMULQ R12, DX
	ADDQ  AX, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12

openAVX2Tail128LoopB:
	ADDQ       $0x10, R9
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x0c, Y9, Y3
	VPSRLD     $0x14, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x07, Y9, Y3
	VPSRLD     $0x19, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPALIGNR   $0x04, Y9, Y9, Y9
	VPALIGNR   $0x08, Y13, Y13, Y13
	VPALIGNR   $0x0c, Y1, Y1, Y1
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x0c, Y9, Y3
	VPSRLD     $0x14, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x07, Y9, Y3
	VPSRLD     $0x19, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPALIGNR   $0x0c, Y9, Y9, Y9
	VPALIGNR   $0x08, Y13, Y13, Y13
	VPALIGNR   $0x04, Y1, Y1, Y1
	CMPQ       R9, CX
	JB         openAVX2Tail128LoopA
	CMPQ       R9, $0xa0
	JNE        openAVX2Tail128LoopB
	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
	VPADDD     32(BP), Y9, Y9
	VPADDD     64(BP), Y13, Y13
	VPADDD     Y4, Y1, Y1
	VPERM2I128 $0x02, Y5, Y9, Y0
	VPERM2I128 $0x02, Y13, Y1, Y14
	VPERM2I128 $0x13, Y5, Y9, Y12
	VPERM2I128 $0x13, Y13, Y1, Y4

openAVX2TailLoop:
	CMPQ BX, $0x20
	JB   openAVX2Tail
	SUBQ $0x20, BX

	// Load for decryption
	VPXOR   (SI), Y0, Y0
	VMOVDQU Y0, (DI)
	LEAQ    32(SI), SI
	LEAQ    32(DI), DI
	VMOVDQA Y14, Y0
	VMOVDQA Y12, Y14
	VMOVDQA Y4, Y12
	JMP     openAVX2TailLoop

openAVX2Tail:
	CMPQ    BX, $0x10
	VMOVDQA X0, X1
	JB      openAVX2TailDone
	SUBQ    $0x10, BX

	// Load for decryption
	VPXOR      (SI), X0, X12
	VMOVDQU    X12, (DI)
	LEAQ       16(SI), SI
	LEAQ       16(DI), DI
	VPERM2I128 $0x11, Y0, Y0, Y0
	VMOVDQA    X0, X1

openAVX2TailDone:
	VZEROUPPER
	JMP openSSETail16

openAVX2Tail256:
	VMOVDQA ·chacha20Constants<>+0(SB), Y0
	VMOVDQA Y0, Y5
	VMOVDQA 32(BP), Y14
	VMOVDQA Y14, Y9
	VMOVDQA 64(BP), Y12
	VMOVDQA Y12, Y13
	VMOVDQA 192(BP), Y4
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
	VMOVDQA Y4, Y7
	VMOVDQA Y1, Y11

	// Compute the number of iterations that will hash data
	MOVQ    BX, 224(BP)
	MOVQ    BX, CX
	SUBQ    $0x80, CX
	SHRQ    $0x04, CX
	MOVQ    $0x0000000a, R9
	CMPQ    CX, $0x0a
	CMOVQGT R9, CX
	MOVQ    SI, BX
	XORQ    R9, R9

openAVX2Tail256LoopA:
	ADDQ  (BX), R10
	ADCQ  8(BX), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), DX
	MOVQ  DX, R15
	MULXQ R10, R13, R14
	IMULQ R12, R15
	MULXQ R11, AX, DX
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), DX
	MULXQ R10, R10, AX
	ADDQ  R10, R14
	MULXQ R11, R11, R8
	ADCQ  R11, R15
	ADCQ  $0x00, R8
	IMULQ R12, DX
	ADDQ  AX, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(BX), BX

openAVX2Tail256LoopB:
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x0c, Y14, Y3
	VPSRLD   $0x14, Y14, Y14
	VPXOR    Y3, Y14, Y14
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x07, Y14, Y3
	VPSRLD   $0x19, Y14, Y14
	VPXOR    Y3, Y14, Y14
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x0c, Y9, Y3
	VPSRLD   $0x14, Y9, Y9
	VPXOR    Y3, Y9, Y9
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x07, Y9, Y3
	VPSRLD   $0x19, Y9, Y9
	VPXOR    Y3, Y9, Y9
	VPALIGNR $0x04, Y14, Y14, Y14
	VPALIGNR $0x04, Y9, Y9, Y9
	VPALIGNR $0x08, Y12, Y12, Y12
	VPALIGNR $0x08, Y13, Y13, Y13
	VPALIGNR $0x0c, Y4, Y4, Y4
	VPALIGNR $0x0c, Y1, Y1, Y1
	INCQ     R9
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x0c, Y14, Y3
	VPSRLD   $0x14, Y14, Y14
	VPXOR    Y3, Y14, Y14
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x07, Y14, Y3
	VPSRLD   $0x19, Y14, Y14
	VPXOR    Y3, Y14, Y14
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x0c, Y9, Y3
	VPSRLD   $0x14, Y9, Y9
	VPXOR    Y3, Y9, Y9
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x07, Y9, Y3
	VPSRLD   $0x19, Y9, Y9
	VPXOR    Y3, Y9, Y9
	VPALIGNR $0x0c, Y14, Y14, Y14
	VPALIGNR $0x0c, Y9, Y9, Y9
	VPALIGNR $0x08, Y12, Y12, Y12
	VPALIGNR $0x08, Y13, Y13, Y13
	VPALIGNR $0x04, Y4, Y4, Y4
	VPALIGNR $0x04, Y1, Y1, Y1
	CMPQ     R9, CX
	JB       openAVX2Tail256LoopA
	CMPQ     R9, $0x0a
	JNE      openAVX2Tail256LoopB
	MOVQ     BX, R9
	SUBQ     SI, BX
	MOVQ     BX, CX
	MOVQ     224(BP), BX

openAVX2Tail256Hash:
	ADDQ  $0x10, CX
	CMPQ  CX, BX
	JGT   openAVX2Tail256HashEnd
	ADDQ  (R9), R10
	ADCQ  8(R9), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), DX
	MOVQ  DX, R15
	MULXQ R10, R13, R14
	IMULQ R12, R15
	MULXQ R11, AX, DX
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), DX
	MULXQ R10, R10, AX
	ADDQ  R10, R14
	MULXQ R11, R11, R8
	ADCQ  R11, R15
	ADCQ  $0x00, R8
	IMULQ R12, DX
	ADDQ  AX, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(R9), R9
	JMP   openAVX2Tail256Hash

openAVX2Tail256HashEnd:
	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
	VPADDD     32(BP), Y14, Y14
	VPADDD     32(BP), Y9, Y9
	VPADDD     64(BP), Y12, Y12
	VPADDD     64(BP), Y13, Y13
	VPADDD     Y7, Y4, Y4
	VPADDD     Y11, Y1, Y1
	VPERM2I128 $0x02, Y0, Y14, Y6
	VPERM2I128 $0x02, Y12, Y4, Y10
	VPERM2I128 $0x13, Y0, Y14, Y8
	VPERM2I128 $0x13, Y12, Y4, Y2
	VPERM2I128 $0x02, Y5, Y9, Y0
	VPERM2I128 $0x02, Y13, Y1, Y14
	VPERM2I128 $0x13, Y5, Y9, Y12
	VPERM2I128 $0x13, Y13, Y1, Y4
	VPXOR      (SI), Y6, Y6
	VPXOR      32(SI), Y10, Y10
	VPXOR      64(SI), Y8, Y8
	VPXOR      96(SI), Y2, Y2
	VMOVDQU    Y6, (DI)
	VMOVDQU    Y10, 32(DI)
	VMOVDQU    Y8, 64(DI)
	VMOVDQU    Y2, 96(DI)
	LEAQ       128(SI), SI
	LEAQ       128(DI), DI
	SUBQ       $0x80, BX
	JMP        openAVX2TailLoop

openAVX2Tail384:
	// Need to decrypt up to 384 bytes - prepare six blocks
	VMOVDQA ·chacha20Constants<>+0(SB), Y0
	VMOVDQA Y0, Y5
	VMOVDQA Y0, Y6
	VMOVDQA 32(BP), Y14
	VMOVDQA Y14, Y9
	VMOVDQA Y14, Y10
	VMOVDQA 64(BP), Y12
	VMOVDQA Y12, Y13
	VMOVDQA Y12, Y8
	VMOVDQA 192(BP), Y4
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
	VMOVDQA Y4, 96(BP)
	VMOVDQA Y1, 128(BP)
	VMOVDQA Y2, 160(BP)

	// Compute the number of iterations that will hash two blocks of data
	MOVQ    BX, 224(BP)
	MOVQ    BX, CX
	SUBQ    $0x00000100, CX
	SHRQ    $0x04, CX
	ADDQ    $0x06, CX
	MOVQ    $0x0000000a, R9
	CMPQ    CX, $0x0a
	CMOVQGT R9, CX
	MOVQ    SI, BX
	XORQ    R9, R9

openAVX2Tail384LoopB:
	ADDQ  (BX), R10
	ADCQ  8(BX), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), DX
	MOVQ  DX, R15
	MULXQ R10, R13, R14
	IMULQ R12, R15
	MULXQ R11, AX, DX
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), DX
	MULXQ R10, R10, AX
	ADDQ  R10, R14
	MULXQ R11, R11, R8
	ADCQ  R11, R15
	ADCQ  $0x00, R8
	IMULQ R12, DX
	ADDQ  AX, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(BX), BX

openAVX2Tail384LoopA:
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x0c, Y14, Y3
	VPSRLD   $0x14, Y14, Y14
	VPXOR    Y3, Y14, Y14
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x07, Y14, Y3
	VPSRLD   $0x19, Y14, Y14
	VPXOR    Y3, Y14, Y14
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x0c, Y9, Y3
	VPSRLD   $0x14, Y9, Y9
	VPXOR    Y3, Y9, Y9
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x07, Y9, Y3
	VPSRLD   $0x19, Y9, Y9
	VPXOR    Y3, Y9, Y9
	VPADDD   Y10, Y6, Y6
	VPXOR    Y6, Y2, Y2
	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
	VPADDD   Y2, Y8, Y8
	VPXOR    Y8, Y10, Y10
	VPSLLD   $0x0c, Y10, Y3
	VPSRLD   $0x14, Y10, Y10
	VPXOR    Y3, Y10, Y10
	VPADDD   Y10, Y6, Y6
	VPXOR    Y6, Y2, Y2
	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
	VPADDD   Y2, Y8, Y8
	VPXOR    Y8, Y10, Y10
	VPSLLD   $0x07, Y10, Y3
	VPSRLD   $0x19, Y10, Y10
	VPXOR    Y3, Y10, Y10
	VPALIGNR $0x04, Y14, Y14, Y14
	VPALIGNR $0x04, Y9, Y9, Y9
	VPALIGNR $0x04, Y10, Y10, Y10
	VPALIGNR $0x08, Y12, Y12, Y12
	VPALIGNR $0x08, Y13, Y13, Y13
	VPALIGNR $0x08, Y8, Y8, Y8
	VPALIGNR $0x0c, Y4, Y4, Y4
	VPALIGNR $0x0c, Y1, Y1, Y1
	VPALIGNR $0x0c, Y2, Y2, Y2
	ADDQ     (BX), R10
	ADCQ     8(BX), R11
	ADCQ     $0x01, R12
	MOVQ     (BP), DX
	MOVQ     DX, R15
	MULXQ    R10, R13, R14
	IMULQ    R12, R15
	MULXQ    R11, AX, DX
	ADDQ     AX, R14
	ADCQ     DX, R15
	MOVQ     8(BP), DX
	MULXQ    R10, R10, AX
	ADDQ     R10, R14
	MULXQ    R11, R11, R8
	ADCQ     R11, R15
	ADCQ     $0x00, R8
	IMULQ    R12, DX
	ADDQ     AX, R15
	ADCQ     DX, R8
	MOVQ     R13, R10
	MOVQ     R14, R11
	MOVQ     R15, R12
	ANDQ     $0x03, R12
	MOVQ     R15, R13
	ANDQ     $-4, R13
	MOVQ     R8, R14
	SHRQ     $0x02, R8, R15
	SHRQ     $0x02, R8
	ADDQ     R13, R10
	ADCQ     R14, R11
	ADCQ     $0x00, R12
	ADDQ     R15, R10
	ADCQ     R8, R11
	ADCQ     $0x00, R12
	LEAQ     16(BX), BX
	INCQ     R9
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x0c, Y14, Y3
	VPSRLD   $0x14, Y14, Y14
	VPXOR    Y3, Y14, Y14
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x07, Y14, Y3
	VPSRLD   $0x19, Y14, Y14
	VPXOR    Y3, Y14, Y14
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x0c, Y9, Y3
	VPSRLD   $0x14, Y9, Y9
	VPXOR    Y3, Y9, Y9
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x07, Y9, Y3
	VPSRLD   $0x19, Y9, Y9
	VPXOR    Y3, Y9, Y9
	VPADDD   Y10, Y6, Y6
	VPXOR    Y6, Y2, Y2
	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
	VPADDD   Y2, Y8, Y8
	VPXOR    Y8, Y10, Y10
	VPSLLD   $0x0c, Y10, Y3
	VPSRLD   $0x14, Y10, Y10
	VPXOR    Y3, Y10, Y10
	VPADDD   Y10, Y6, Y6
	VPXOR    Y6, Y2, Y2
	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
	VPADDD   Y2, Y8, Y8
	VPXOR    Y8, Y10, Y10
	VPSLLD   $0x07, Y10, Y3
	VPSRLD   $0x19, Y10, Y10
	VPXOR    Y3, Y10, Y10
	VPALIGNR $0x0c, Y14, Y14, Y14
	VPALIGNR $0x0c, Y9, Y9, Y9
	VPALIGNR $0x0c, Y10, Y10, Y10
	VPALIGNR $0x08, Y12, Y12, Y12
	VPALIGNR $0x08, Y13, Y13, Y13
	VPALIGNR $0x08, Y8, Y8, Y8
	VPALIGNR $0x04, Y4, Y4, Y4
	VPALIGNR $0x04, Y1, Y1, Y1
	VPALIGNR $0x04, Y2, Y2, Y2
	CMPQ     R9, CX
	JB       openAVX2Tail384LoopB
	CMPQ     R9, $0x0a
	JNE      openAVX2Tail384LoopA
	MOVQ     BX, R9
	SUBQ     SI, BX
	MOVQ     BX, CX
	MOVQ     224(BP), BX

openAVX2Tail384Hash:
	ADDQ  $0x10, CX
	CMPQ  CX, BX
	JGT   openAVX2Tail384HashEnd
	ADDQ  (R9), R10
	ADCQ  8(R9), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), DX
	MOVQ  DX, R15
	MULXQ R10, R13, R14
	IMULQ R12, R15
	MULXQ R11, AX, DX
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), DX
	MULXQ R10, R10, AX
	ADDQ  R10, R14
	MULXQ R11, R11, R8
	ADCQ  R11, R15
	ADCQ  $0x00, R8
	IMULQ R12, DX
	ADDQ  AX, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(R9), R9
	JMP   openAVX2Tail384Hash

openAVX2Tail384HashEnd:
	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
	VPADDD     32(BP), Y14, Y14
	VPADDD     32(BP), Y9, Y9
	VPADDD     32(BP), Y10, Y10
	VPADDD     64(BP), Y12, Y12
	VPADDD     64(BP), Y13, Y13
	VPADDD     64(BP), Y8, Y8
	VPADDD     96(BP), Y4, Y4
	VPADDD     128(BP), Y1, Y1
	VPADDD     160(BP), Y2, Y2
	VPERM2I128 $0x02, Y0, Y14, Y3
	VPERM2I128 $0x02, Y12, Y4, Y7
	VPERM2I128 $0x13, Y0, Y14, Y11
	VPERM2I128 $0x13, Y12, Y4, Y15
	VPXOR      (SI), Y3, Y3
	VPXOR      32(SI), Y7, Y7
	VPXOR      64(SI), Y11, Y11
	VPXOR      96(SI), Y15, Y15
	VMOVDQU    Y3, (DI)
	VMOVDQU    Y7, 32(DI)
	VMOVDQU    Y11, 64(DI)
	VMOVDQU    Y15, 96(DI)
	VPERM2I128 $0x02, Y5, Y9, Y3
	VPERM2I128 $0x02, Y13, Y1, Y7
	VPERM2I128 $0x13, Y5, Y9, Y11
	VPERM2I128 $0x13, Y13, Y1, Y15
	VPXOR      128(SI), Y3, Y3
	VPXOR      160(SI), Y7, Y7
	VPXOR      192(SI), Y11, Y11
	VPXOR      224(SI), Y15, Y15
	VMOVDQU    Y3, 128(DI)
	VMOVDQU    Y7, 160(DI)
	VMOVDQU    Y11, 192(DI)
	VMOVDQU    Y15, 224(DI)
	VPERM2I128 $0x02, Y6, Y10, Y0
	VPERM2I128 $0x02, Y8, Y2, Y14
	VPERM2I128 $0x13, Y6, Y10, Y12
	VPERM2I128 $0x13, Y8, Y2, Y4
	LEAQ       256(SI), SI
	LEAQ       256(DI), DI
	SUBQ       $0x00000100, BX
	JMP        openAVX2TailLoop

openAVX2Tail512:
	VMOVDQU ·chacha20Constants<>+0(SB), Y0
	VMOVDQA Y0, Y5
	VMOVDQA Y0, Y6
	VMOVDQA Y0, Y7
	VMOVDQA 32(BP), Y14
	VMOVDQA Y14, Y9
	VMOVDQA Y14, Y10
	VMOVDQA Y14, Y11
	VMOVDQA 64(BP), Y12
	VMOVDQA Y12, Y13
	VMOVDQA Y12, Y8
	VMOVDQA Y12, Y15
	VMOVDQA 192(BP), Y4
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
	VMOVDQA Y4, 96(BP)
	VMOVDQA Y1, 128(BP)
	VMOVDQA Y2, 160(BP)
	VMOVDQA Y3, 192(BP)
	XORQ    CX, CX
	MOVQ    SI, R9

openAVX2Tail512LoopB:
	ADDQ  (R9), R10
	ADCQ  8(R9), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), DX
	MOVQ  DX, R15
	MULXQ R10, R13, R14
	IMULQ R12, R15
	MULXQ R11, AX, DX
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), DX
	MULXQ R10, R10, AX
	ADDQ  R10, R14
	MULXQ R11, R11, R8
	ADCQ  R11, R15
	ADCQ  $0x00, R8
	IMULQ R12, DX
	ADDQ  AX, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(R9), R9

openAVX2Tail512LoopA:
	VPADDD   Y14, Y0, Y0
	VPADDD   Y9, Y5, Y5
	VPADDD   Y10, Y6, Y6
	VPADDD   Y11, Y7, Y7
	VPXOR    Y0, Y4, Y4
	VPXOR    Y5, Y1, Y1
	VPXOR    Y6, Y2, Y2
	VPXOR    Y7, Y3, Y3
	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
	VPADDD   Y4, Y12, Y12
	VPADDD   Y1, Y13, Y13
	VPADDD   Y2, Y8, Y8
	VPADDD   Y3, Y15, Y15
	VPXOR    Y12, Y14, Y14
	VPXOR    Y13, Y9, Y9
	VPXOR    Y8, Y10, Y10
	VPXOR    Y15, Y11, Y11
	VMOVDQA  Y15, 224(BP)
	VPSLLD   $0x0c, Y14, Y15
	VPSRLD   $0x14, Y14, Y14
	VPXOR    Y15, Y14, Y14
	VPSLLD   $0x0c, Y9, Y15
	VPSRLD   $0x14, Y9, Y9
	VPXOR    Y15, Y9, Y9
	VPSLLD   $0x0c, Y10, Y15
	VPSRLD   $0x14, Y10, Y10
	VPXOR    Y15, Y10, Y10
	VPSLLD   $0x0c, Y11, Y15
	VPSRLD   $0x14, Y11, Y11
	VPXOR    Y15, Y11, Y11
	VMOVDQA  224(BP), Y15
	ADDQ     (R9), R10
	ADCQ     8(R9), R11
	ADCQ     $0x01, R12
	MOVQ     (BP), DX
	MOVQ     DX, R15
	MULXQ    R10, R13, R14
	IMULQ    R12, R15
	MULXQ    R11, AX, DX
	ADDQ     AX, R14
	ADCQ     DX, R15
	MOVQ     8(BP), DX
	MULXQ    R10, R10, AX
	ADDQ     R10, R14
	MULXQ    R11, R11, R8
	ADCQ     R11, R15
	ADCQ     $0x00, R8
	IMULQ    R12, DX
	ADDQ     AX, R15
	ADCQ     DX, R8
	MOVQ     R13, R10
	MOVQ     R14, R11
	MOVQ     R15, R12
	ANDQ     $0x03, R12
	MOVQ     R15, R13
	ANDQ     $-4, R13
	MOVQ     R8, R14
	SHRQ     $0x02, R8, R15
	SHRQ     $0x02, R8
	ADDQ     R13, R10
	ADCQ     R14, R11
	ADCQ     $0x00, R12
	ADDQ     R15, R10
	ADCQ     R8, R11
	ADCQ     $0x00, R12
	VPADDD   Y14, Y0, Y0
	VPADDD   Y9, Y5, Y5
	VPADDD   Y10, Y6, Y6
	VPADDD   Y11, Y7, Y7
	VPXOR    Y0, Y4, Y4
	VPXOR    Y5, Y1, Y1
	VPXOR    Y6, Y2, Y2
	VPXOR    Y7, Y3, Y3
	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
	VPADDD   Y4, Y12, Y12
	VPADDD   Y1, Y13, Y13
	VPADDD   Y2, Y8, Y8
	VPADDD   Y3, Y15, Y15
	VPXOR    Y12, Y14, Y14
	VPXOR    Y13, Y9, Y9
	VPXOR    Y8, Y10, Y10
	VPXOR    Y15, Y11, Y11
	VMOVDQA  Y15, 224(BP)
	VPSLLD   $0x07, Y14, Y15
	VPSRLD   $0x19, Y14, Y14
	VPXOR    Y15, Y14, Y14
	VPSLLD   $0x07, Y9, Y15
	VPSRLD   $0x19, Y9, Y9
	VPXOR    Y15, Y9, Y9
	VPSLLD   $0x07, Y10, Y15
	VPSRLD   $0x19, Y10, Y10
	VPXOR    Y15, Y10, Y10
	VPSLLD   $0x07, Y11, Y15
	VPSRLD   $0x19, Y11, Y11
	VPXOR    Y15, Y11, Y11
	VMOVDQA  224(BP), Y15
	VPALIGNR $0x04, Y14, Y14, Y14
	VPALIGNR $0x04, Y9, Y9, Y9
	VPALIGNR $0x04, Y10, Y10, Y10
	VPALIGNR $0x04, Y11, Y11, Y11
	VPALIGNR $0x08, Y12, Y12, Y12
	VPALIGNR $0x08, Y13, Y13, Y13
	VPALIGNR $0x08, Y8, Y8, Y8
	VPALIGNR $0x08, Y15, Y15, Y15
	VPALIGNR $0x0c, Y4, Y4, Y4
	VPALIGNR $0x0c, Y1, Y1, Y1
	VPALIGNR $0x0c, Y2, Y2, Y2
	VPALIGNR $0x0c, Y3, Y3, Y3
	VPADDD   Y14, Y0, Y0
	VPADDD   Y9, Y5, Y5
	VPADDD   Y10, Y6, Y6
	VPADDD   Y11, Y7, Y7
	VPXOR    Y0, Y4, Y4
	VPXOR    Y5, Y1, Y1
	VPXOR    Y6, Y2, Y2
	VPXOR    Y7, Y3, Y3
	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
	VPADDD   Y4, Y12, Y12
	VPADDD   Y1, Y13, Y13
	VPADDD   Y2, Y8, Y8
	VPADDD   Y3, Y15, Y15
	VPXOR    Y12, Y14, Y14
	VPXOR    Y13, Y9, Y9
	VPXOR    Y8, Y10, Y10
	VPXOR    Y15, Y11, Y11
	ADDQ     16(R9), R10
	ADCQ     24(R9), R11
	ADCQ     $0x01, R12
	MOVQ     (BP), DX
	MOVQ     DX, R15
	MULXQ    R10, R13, R14
	IMULQ    R12, R15
	MULXQ    R11, AX, DX
	ADDQ     AX, R14
	ADCQ     DX, R15
	MOVQ     8(BP), DX
	MULXQ    R10, R10, AX
	ADDQ     R10, R14
	MULXQ    R11, R11, R8
	ADCQ     R11, R15
	ADCQ     $0x00, R8
	IMULQ    R12, DX
	ADDQ     AX, R15
	ADCQ     DX, R8
	MOVQ     R13, R10
	MOVQ     R14, R11
	MOVQ     R15, R12
	ANDQ     $0x03, R12
	MOVQ     R15, R13
	ANDQ     $-4, R13
	MOVQ     R8, R14
	SHRQ     $0x02, R8, R15
	SHRQ     $0x02, R8
	ADDQ     R13, R10
	ADCQ     R14, R11
	ADCQ     $0x00, R12
	ADDQ     R15, R10
	ADCQ     R8, R11
	ADCQ     $0x00, R12
	LEAQ     32(R9), R9
	VMOVDQA  Y15, 224(BP)
	VPSLLD   $0x0c, Y14, Y15
	VPSRLD   $0x14, Y14, Y14
	VPXOR    Y15, Y14, Y14
	VPSLLD   $0x0c, Y9, Y15
	VPSRLD   $0x14, Y9, Y9
	VPXOR    Y15, Y9, Y9
	VPSLLD   $0x0c, Y10, Y15
	VPSRLD   $0x14, Y10, Y10
	VPXOR    Y15, Y10, Y10
	VPSLLD   $0x0c, Y11, Y15
	VPSRLD   $0x14, Y11, Y11
	VPXOR    Y15, Y11, Y11
	VMOVDQA  224(BP), Y15
	VPADDD   Y14, Y0, Y0
	VPADDD   Y9, Y5, Y5
	VPADDD   Y10, Y6, Y6
	VPADDD   Y11, Y7, Y7
	VPXOR    Y0, Y4, Y4
	VPXOR    Y5, Y1, Y1
	VPXOR    Y6, Y2, Y2
	VPXOR    Y7, Y3, Y3
	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
	VPADDD   Y4, Y12, Y12
	VPADDD   Y1, Y13, Y13
	VPADDD   Y2, Y8, Y8
	VPADDD   Y3, Y15, Y15
	VPXOR    Y12, Y14, Y14
	VPXOR    Y13, Y9, Y9
	VPXOR    Y8, Y10, Y10
	VPXOR    Y15, Y11, Y11
	VMOVDQA  Y15, 224(BP)
	VPSLLD   $0x07, Y14, Y15
	VPSRLD   $0x19, Y14, Y14
	VPXOR    Y15, Y14, Y14
	VPSLLD   $0x07, Y9, Y15
	VPSRLD   $0x19, Y9, Y9
	VPXOR    Y15, Y9, Y9
	VPSLLD   $0x07, Y10, Y15
	VPSRLD   $0x19, Y10, Y10
	VPXOR    Y15, Y10, Y10
	VPSLLD   $0x07, Y11, Y15
	VPSRLD   $0x19, Y11, Y11
	VPXOR    Y15, Y11, Y11
	VMOVDQA  224(BP), Y15
	VPALIGNR $0x0c, Y14, Y14, Y14
	VPALIGNR $0x0c, Y9, Y9, Y9
	VPALIGNR $0x0c, Y10, Y10, Y10
	VPALIGNR $0x0c, Y11, Y11, Y11
	VPALIGNR $0x08, Y12, Y12, Y12
	VPALIGNR $0x08, Y13, Y13, Y13
	VPALIGNR $0x08, Y8, Y8, Y8
	VPALIGNR $0x08, Y15, Y15, Y15
	VPALIGNR $0x04, Y4, Y4, Y4
	VPALIGNR $0x04, Y1, Y1, Y1
	VPALIGNR $0x04, Y2, Y2, Y2
	VPALIGNR $0x04, Y3, Y3, Y3
	INCQ     CX
	CMPQ     CX, $0x04
	JLT      openAVX2Tail512LoopB
	CMPQ     CX, $0x0a
	JNE      openAVX2Tail512LoopA
	MOVQ     BX, CX
	SUBQ     $0x00000180, CX
	ANDQ     $-16, CX

openAVX2Tail512HashLoop:
	TESTQ CX, CX
	JE    openAVX2Tail512HashEnd
	ADDQ  (R9), R10
	ADCQ  8(R9), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), DX
	MOVQ  DX, R15
	MULXQ R10, R13, R14
	IMULQ R12, R15
	MULXQ R11, AX, DX
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), DX
	MULXQ R10, R10, AX
	ADDQ  R10, R14
	MULXQ R11, R11, R8
	ADCQ  R11, R15
	ADCQ  $0x00, R8
	IMULQ R12, DX
	ADDQ  AX, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(R9), R9
	SUBQ  $0x10, CX
	JMP   openAVX2Tail512HashLoop

openAVX2Tail512HashEnd:
	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
	VPADDD     ·chacha20Constants<>+0(SB), Y7, Y7
	VPADDD     32(BP), Y14, Y14
	VPADDD     32(BP), Y9, Y9
	VPADDD     32(BP), Y10, Y10
	VPADDD     32(BP), Y11, Y11
	VPADDD     64(BP), Y12, Y12
	VPADDD     64(BP), Y13, Y13
	VPADDD     64(BP), Y8, Y8
	VPADDD     64(BP), Y15, Y15
	VPADDD     96(BP), Y4, Y4
	VPADDD     128(BP), Y1, Y1
	VPADDD     160(BP), Y2, Y2
	VPADDD     192(BP), Y3, Y3
	VMOVDQA    Y15, 224(BP)
	VPERM2I128 $0x02, Y0, Y14, Y15
	VPERM2I128 $0x13, Y0, Y14, Y14
	VPERM2I128 $0x02, Y12, Y4, Y0
	VPERM2I128 $0x13, Y12, Y4, Y12
	VPXOR      (SI), Y15, Y15
	VPXOR      32(SI), Y0, Y0
	VPXOR      64(SI), Y14, Y14
	VPXOR      96(SI), Y12, Y12
	VMOVDQU    Y15, (DI)
	VMOVDQU    Y0, 32(DI)
	VMOVDQU    Y14, 64(DI)
	VMOVDQU    Y12, 96(DI)
	VPERM2I128 $0x02, Y5, Y9, Y0
	VPERM2I128 $0x02, Y13, Y1, Y14
	VPERM2I128 $0x13, Y5, Y9, Y12
	VPERM2I128 $0x13, Y13, Y1, Y4
	VPXOR      128(SI), Y0, Y0
	VPXOR      160(SI), Y14, Y14
	VPXOR      192(SI), Y12, Y12
	VPXOR      224(SI), Y4, Y4
	VMOVDQU    Y0, 128(DI)
	VMOVDQU    Y14, 160(DI)
	VMOVDQU    Y12, 192(DI)
	VMOVDQU    Y4, 224(DI)
	VPERM2I128 $0x02, Y6, Y10, Y0
	VPERM2I128 $0x02, Y8, Y2, Y14
	VPERM2I128 $0x13, Y6, Y10, Y12
	VPERM2I128 $0x13, Y8, Y2, Y4
	VPXOR      256(SI), Y0, Y0
	VPXOR      288(SI), Y14, Y14
	VPXOR      320(SI), Y12, Y12
	VPXOR      352(SI), Y4, Y4
	VMOVDQU    Y0, 256(DI)
	VMOVDQU    Y14, 288(DI)
	VMOVDQU    Y12, 320(DI)
	VMOVDQU    Y4, 352(DI)
	VPERM2I128 $0x02, Y7, Y11, Y0
	VPERM2I128 $0x02, 224(BP), Y3, Y14
	VPERM2I128 $0x13, Y7, Y11, Y12
	VPERM2I128 $0x13, 224(BP), Y3, Y4
	LEAQ       384(SI), SI
	LEAQ       384(DI), DI
	SUBQ       $0x00000180, BX
	JMP        openAVX2TailLoop

DATA ·chacha20Constants<>+0(SB)/4, $0x61707865
DATA ·chacha20Constants<>+4(SB)/4, $0x3320646e
DATA ·chacha20Constants<>+8(SB)/4, $0x79622d32
DATA ·chacha20Constants<>+12(SB)/4, $0x6b206574
DATA ·chacha20Constants<>+16(SB)/4, $0x61707865
DATA ·chacha20Constants<>+20(SB)/4, $0x3320646e
DATA ·chacha20Constants<>+24(SB)/4, $0x79622d32
DATA ·chacha20Constants<>+28(SB)/4, $0x6b206574
GLOBL ·chacha20Constants<>(SB), RODATA|NOPTR, $32

DATA ·polyClampMask<>+0(SB)/8, $0x0ffffffc0fffffff
DATA ·polyClampMask<>+8(SB)/8, $0x0ffffffc0ffffffc
DATA ·polyClampMask<>+16(SB)/8, $0xffffffffffffffff
DATA ·polyClampMask<>+24(SB)/8, $0xffffffffffffffff
GLOBL ·polyClampMask<>(SB), RODATA|NOPTR, $32

DATA ·sseIncMask<>+0(SB)/8, $0x0000000000000001
DATA ·sseIncMask<>+8(SB)/8, $0x0000000000000000
GLOBL ·sseIncMask<>(SB), RODATA|NOPTR, $16

DATA ·andMask<>+0(SB)/8, $0x00000000000000ff
DATA ·andMask<>+8(SB)/8, $0x0000000000000000
DATA ·andMask<>+16(SB)/8, $0x000000000000ffff
DATA ·andMask<>+24(SB)/8, $0x0000000000000000
DATA ·andMask<>+32(SB)/8, $0x0000000000ffffff
DATA ·andMask<>+40(SB)/8, $0x0000000000000000
DATA ·andMask<>+48(SB)/8, $0x00000000ffffffff
DATA ·andMask<>+56(SB)/8, $0x0000000000000000
DATA ·andMask<>+64(SB)/8, $0x000000ffffffffff
DATA ·andMask<>+72(SB)/8, $0x0000000000000000
DATA ·andMask<>+80(SB)/8, $0x0000ffffffffffff
DATA ·andMask<>+88(SB)/8, $0x0000000000000000
DATA ·andMask<>+96(SB)/8, $0x00ffffffffffffff
DATA ·andMask<>+104(SB)/8, $0x0000000000000000
DATA ·andMask<>+112(SB)/8, $0xffffffffffffffff
DATA ·andMask<>+120(SB)/8, $0x0000000000000000
DATA ·andMask<>+128(SB)/8, $0xffffffffffffffff
DATA ·andMask<>+136(SB)/8, $0x00000000000000ff
DATA ·andMask<>+144(SB)/8, $0xffffffffffffffff
DATA ·andMask<>+152(SB)/8, $0x000000000000ffff
DATA ·andMask<>+160(SB)/8, $0xffffffffffffffff
DATA ·andMask<>+168(SB)/8, $0x0000000000ffffff
DATA ·andMask<>+176(SB)/8, $0xffffffffffffffff
DATA ·andMask<>+184(SB)/8, $0x00000000ffffffff
DATA ·andMask<>+192(SB)/8, $0xffffffffffffffff
DATA ·andMask<>+200(SB)/8, $0x000000ffffffffff
DATA ·andMask<>+208(SB)/8, $0xffffffffffffffff
DATA ·andMask<>+216(SB)/8, $0x0000ffffffffffff
DATA ·andMask<>+224(SB)/8, $0xffffffffffffffff
DATA ·andMask<>+232(SB)/8, $0x00ffffffffffffff
GLOBL ·andMask<>(SB), RODATA|NOPTR, $240

DATA ·avx2InitMask<>+0(SB)/8, $0x0000000000000000
DATA ·avx2InitMask<>+8(SB)/8, $0x0000000000000000
DATA ·avx2InitMask<>+16(SB)/8, $0x0000000000000001
DATA ·avx2InitMask<>+24(SB)/8, $0x0000000000000000
GLOBL ·avx2InitMask<>(SB), RODATA|NOPTR, $32

DATA ·rol16<>+0(SB)/8, $0x0504070601000302
DATA ·rol16<>+8(SB)/8, $0x0d0c0f0e09080b0a
DATA ·rol16<>+16(SB)/8, $0x0504070601000302
DATA ·rol16<>+24(SB)/8, $0x0d0c0f0e09080b0a
GLOBL ·rol16<>(SB), RODATA|NOPTR, $32

DATA ·rol8<>+0(SB)/8, $0x0605040702010003
DATA ·rol8<>+8(SB)/8, $0x0e0d0c0f0a09080b
DATA ·rol8<>+16(SB)/8, $0x0605040702010003
DATA ·rol8<>+24(SB)/8, $0x0e0d0c0f0a09080b
GLOBL ·rol8<>(SB), RODATA|NOPTR, $32

DATA ·avx2IncMask<>+0(SB)/8, $0x0000000000000002
DATA ·avx2IncMask<>+8(SB)/8, $0x0000000000000000
DATA ·avx2IncMask<>+16(SB)/8, $0x0000000000000002
DATA ·avx2IncMask<>+24(SB)/8, $0x0000000000000000
GLOBL ·avx2IncMask<>(SB), RODATA|NOPTR, $32

// func chacha20Poly1305Seal(dst []byte, key []uint32, src []byte, ad []byte)
// Requires: AVX, AVX2, BMI2, CMOV, SSE2
TEXT ·chacha20Poly1305Seal(SB), $288-96
	MOVQ SP, BP
	ADDQ $0x20, BP
	ANDQ $-32, BP
	MOVQ dst_base+0(FP), DI
	MOVQ key_base+24(FP), R8
	MOVQ src_base+48(FP), SI
	MOVQ src_len+56(FP), BX
	MOVQ ad_base+72(FP), CX
	CMPB ·useAVX2+0(SB), $0x01
	JE   chacha20Poly1305Seal_AVX2

	// Special optimization, for very short buffers
	CMPQ BX, $0x80
	JBE  sealSSE128

	// In the seal case - prepare the poly key + 3 blocks of stream in the first iteration
	MOVOU ·chacha20Constants<>+0(SB), X0
	MOVOU 16(R8), X3
	MOVOU 32(R8), X6
	MOVOU 48(R8), X9

	// Store state on stack for future use
	MOVO X3, 32(BP)
	MOVO X6, 48(BP)

	// Load state, increment counter blocks
	MOVO  X0, X1
	MOVO  X3, X4
	MOVO  X6, X7
	MOVO  X9, X10
	PADDL ·sseIncMask<>+0(SB), X10
	MOVO  X1, X2
	MOVO  X4, X5
	MOVO  X7, X8
	MOVO  X10, X11
	PADDL ·sseIncMask<>+0(SB), X11
	MOVO  X2, X12
	MOVO  X5, X13
	MOVO  X8, X14
	MOVO  X11, X15
	PADDL ·sseIncMask<>+0(SB), X15

	// Store counters
	MOVO X9, 80(BP)
	MOVO X10, 96(BP)
	MOVO X11, 112(BP)
	MOVO X15, 128(BP)
	MOVQ $0x0000000a, R9

sealSSEIntroLoop:
	MOVO  X14, 64(BP)
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X14)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X3
	PXOR  X14, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X14)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X3
	PXOR  X14, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X14)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X4
	PXOR  X14, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X14)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X4
	PXOR  X14, X4
	PADDD X5, X2
	PXOR  X2, X11
	ROL16(X11, X14)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X5
	PXOR  X14, X5
	PADDD X5, X2
	PXOR  X2, X11
	ROL8(X11, X14)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X5
	PXOR  X14, X5
	MOVO  64(BP), X14
	MOVO  X7, 64(BP)
	PADDD X13, X12
	PXOR  X12, X15
	ROL16(X15, X7)
	PADDD X15, X14
	PXOR  X14, X13
	MOVO  X13, X7
	PSLLL $0x0c, X7
	PSRLL $0x14, X13
	PXOR  X7, X13
	PADDD X13, X12
	PXOR  X12, X15
	ROL8(X15, X7)
	PADDD X15, X14
	PXOR  X14, X13
	MOVO  X13, X7
	PSLLL $0x07, X7
	PSRLL $0x19, X13
	PXOR  X7, X13
	MOVO  64(BP), X7
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc0
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x0c
	MOVO  X14, 64(BP)
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X14)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X3
	PXOR  X14, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X14)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X3
	PXOR  X14, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X14)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X4
	PXOR  X14, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X14)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X4
	PXOR  X14, X4
	PADDD X5, X2
	PXOR  X2, X11
	ROL16(X11, X14)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X5
	PXOR  X14, X5
	PADDD X5, X2
	PXOR  X2, X11
	ROL8(X11, X14)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X5
	PXOR  X14, X5
	MOVO  64(BP), X14
	MOVO  X7, 64(BP)
	PADDD X13, X12
	PXOR  X12, X15
	ROL16(X15, X7)
	PADDD X15, X14
	PXOR  X14, X13
	MOVO  X13, X7
	PSLLL $0x0c, X7
	PSRLL $0x14, X13
	PXOR  X7, X13
	PADDD X13, X12
	PXOR  X12, X15
	ROL8(X15, X7)
	PADDD X15, X14
	PXOR  X14, X13
	MOVO  X13, X7
	PSLLL $0x07, X7
	PSRLL $0x19, X13
	PXOR  X7, X13
	MOVO  64(BP), X7
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc0
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x04
	DECQ  R9
	JNE   sealSSEIntroLoop

	// Add in the state
	PADDD ·chacha20Constants<>+0(SB), X0
	PADDD ·chacha20Constants<>+0(SB), X1
	PADDD ·chacha20Constants<>+0(SB), X2
	PADDD ·chacha20Constants<>+0(SB), X12
	PADDD 32(BP), X3
	PADDD 32(BP), X4
	PADDD 32(BP), X5
	PADDD 32(BP), X13
	PADDD 48(BP), X7
	PADDD 48(BP), X8
	PADDD 48(BP), X14
	PADDD 96(BP), X10
	PADDD 112(BP), X11
	PADDD 128(BP), X15

	// Clamp and store the key
	PAND ·polyClampMask<>+0(SB), X0
	MOVO X0, (BP)
	MOVO X3, 16(BP)

	// Hash AAD
	MOVQ  ad_len+80(FP), R9
	CALL  polyHashADInternal<>(SB)
	MOVOU (SI), X0
	MOVOU 16(SI), X3
	MOVOU 32(SI), X6
	MOVOU 48(SI), X9
	PXOR  X0, X1
	PXOR  X3, X4
	PXOR  X6, X7
	PXOR  X9, X10
	MOVOU X1, (DI)
	MOVOU X4, 16(DI)
	MOVOU X7, 32(DI)
	MOVOU X10, 48(DI)
	MOVOU 64(SI), X0
	MOVOU 80(SI), X3
	MOVOU 96(SI), X6
	MOVOU 112(SI), X9
	PXOR  X0, X2
	PXOR  X3, X5
	PXOR  X6, X8
	PXOR  X9, X11
	MOVOU X2, 64(DI)
	MOVOU X5, 80(DI)
	MOVOU X8, 96(DI)
	MOVOU X11, 112(DI)
	MOVQ  $0x00000080, CX
	SUBQ  $0x80, BX
	LEAQ  128(SI), SI
	MOVO  X12, X1
	MOVO  X13, X4
	MOVO  X14, X7
	MOVO  X15, X10
	CMPQ  BX, $0x40
	JBE   sealSSE128SealHash
	MOVOU (SI), X0
	MOVOU 16(SI), X3
	MOVOU 32(SI), X6
	MOVOU 48(SI), X9
	PXOR  X0, X12
	PXOR  X3, X13
	PXOR  X6, X14
	PXOR  X9, X15
	MOVOU X12, 128(DI)
	MOVOU X13, 144(DI)
	MOVOU X14, 160(DI)
	MOVOU X15, 176(DI)
	ADDQ  $0x40, CX
	SUBQ  $0x40, BX
	LEAQ  64(SI), SI
	MOVQ  $0x00000002, CX
	MOVQ  $0x00000008, R9
	CMPQ  BX, $0x40
	JBE   sealSSETail64
	CMPQ  BX, $0x80
	JBE   sealSSETail128
	CMPQ  BX, $0xc0
	JBE   sealSSETail192

sealSSEMainLoop:
	// Load state, increment counter blocks
	MOVO  ·chacha20Constants<>+0(SB), X0
	MOVO  32(BP), X3
	MOVO  48(BP), X6
	MOVO  128(BP), X9
	PADDL ·sseIncMask<>+0(SB), X9
	MOVO  X0, X1
	MOVO  X3, X4
	MOVO  X6, X7
	MOVO  X9, X10
	PADDL ·sseIncMask<>+0(SB), X10
	MOVO  X1, X2
	MOVO  X4, X5
	MOVO  X7, X8
	MOVO  X10, X11
	PADDL ·sseIncMask<>+0(SB), X11
	MOVO  X2, X12
	MOVO  X5, X13
	MOVO  X8, X14
	MOVO  X11, X15
	PADDL ·sseIncMask<>+0(SB), X15

	// Store counters
	MOVO X9, 80(BP)
	MOVO X10, 96(BP)
	MOVO X11, 112(BP)
	MOVO X15, 128(BP)

sealSSEInnerLoop:
	MOVO  X14, 64(BP)
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X14)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X3
	PXOR  X14, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X14)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X3
	PXOR  X14, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X14)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X4
	PXOR  X14, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X14)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X4
	PXOR  X14, X4
	PADDD X5, X2
	PXOR  X2, X11
	ROL16(X11, X14)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X5
	PXOR  X14, X5
	PADDD X5, X2
	PXOR  X2, X11
	ROL8(X11, X14)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X5
	PXOR  X14, X5
	MOVO  64(BP), X14
	MOVO  X7, 64(BP)
	PADDD X13, X12
	PXOR  X12, X15
	ROL16(X15, X7)
	PADDD X15, X14
	PXOR  X14, X13
	MOVO  X13, X7
	PSLLL $0x0c, X7
	PSRLL $0x14, X13
	PXOR  X7, X13
	PADDD X13, X12
	PXOR  X12, X15
	ROL8(X15, X7)
	PADDD X15, X14
	PXOR  X14, X13
	MOVO  X13, X7
	PSLLL $0x07, X7
	PSRLL $0x19, X13
	PXOR  X7, X13
	MOVO  64(BP), X7
	ADDQ  (DI), R10
	ADCQ  8(DI), R11
	ADCQ  $0x01, R12
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc0
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x0c
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	LEAQ  16(DI), DI
	MOVO  X14, 64(BP)
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X14)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X3
	PXOR  X14, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X14)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X3
	PXOR  X14, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X14)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X4
	PXOR  X14, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X14)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X4
	PXOR  X14, X4
	PADDD X5, X2
	PXOR  X2, X11
	ROL16(X11, X14)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X14
	PSLLL $0x0c, X14
	PSRLL $0x14, X5
	PXOR  X14, X5
	PADDD X5, X2
	PXOR  X2, X11
	ROL8(X11, X14)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X14
	PSLLL $0x07, X14
	PSRLL $0x19, X5
	PXOR  X14, X5
	MOVO  64(BP), X14
	MOVO  X7, 64(BP)
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	PADDD X13, X12
	PXOR  X12, X15
	ROL16(X15, X7)
	PADDD X15, X14
	PXOR  X14, X13
	MOVO  X13, X7
	PSLLL $0x0c, X7
	PSRLL $0x14, X13
	PXOR  X7, X13
	PADDD X13, X12
	PXOR  X12, X15
	ROL8(X15, X7)
	PADDD X15, X14
	PXOR  X14, X13
	MOVO  X13, X7
	PSLLL $0x07, X7
	PSRLL $0x19, X13
	PXOR  X7, X13
	MOVO  64(BP), X7
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc0
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x04
	DECQ  R9
	JGE   sealSSEInnerLoop
	ADDQ  (DI), R10
	ADCQ  8(DI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(DI), DI
	DECQ  CX
	JG    sealSSEInnerLoop

	// Add in the state
	PADDD ·chacha20Constants<>+0(SB), X0
	PADDD ·chacha20Constants<>+0(SB), X1
	PADDD ·chacha20Constants<>+0(SB), X2
	PADDD ·chacha20Constants<>+0(SB), X12
	PADDD 32(BP), X3
	PADDD 32(BP), X4
	PADDD 32(BP), X5
	PADDD 32(BP), X13
	PADDD 48(BP), X6
	PADDD 48(BP), X7
	PADDD 48(BP), X8
	PADDD 48(BP), X14
	PADDD 80(BP), X9
	PADDD 96(BP), X10
	PADDD 112(BP), X11
	PADDD 128(BP), X15
	MOVO  X15, 64(BP)

	// Load - xor - store
	MOVOU (SI), X15
	PXOR  X15, X0
	MOVOU 16(SI), X15
	PXOR  X15, X3
	MOVOU 32(SI), X15
	PXOR  X15, X6
	MOVOU 48(SI), X15
	PXOR  X15, X9
	MOVOU X0, (DI)
	MOVOU X3, 16(DI)
	MOVOU X6, 32(DI)
	MOVOU X9, 48(DI)
	MOVO  64(BP), X15
	MOVOU 64(SI), X0
	MOVOU 80(SI), X3
	MOVOU 96(SI), X6
	MOVOU 112(SI), X9
	PXOR  X0, X1
	PXOR  X3, X4
	PXOR  X6, X7
	PXOR  X9, X10
	MOVOU X1, 64(DI)
	MOVOU X4, 80(DI)
	MOVOU X7, 96(DI)
	MOVOU X10, 112(DI)
	MOVOU 128(SI), X0
	MOVOU 144(SI), X3
	MOVOU 160(SI), X6
	MOVOU 176(SI), X9
	PXOR  X0, X2
	PXOR  X3, X5
	PXOR  X6, X8
	PXOR  X9, X11
	MOVOU X2, 128(DI)
	MOVOU X5, 144(DI)
	MOVOU X8, 160(DI)
	MOVOU X11, 176(DI)
	ADDQ  $0xc0, SI
	MOVQ  $0x000000c0, CX
	SUBQ  $0xc0, BX
	MOVO  X12, X1
	MOVO  X13, X4
	MOVO  X14, X7
	MOVO  X15, X10
	CMPQ  BX, $0x40
	JBE   sealSSE128SealHash
	MOVOU (SI), X0
	MOVOU 16(SI), X3
	MOVOU 32(SI), X6
	MOVOU 48(SI), X9
	PXOR  X0, X12
	PXOR  X3, X13
	PXOR  X6, X14
	PXOR  X9, X15
	MOVOU X12, 192(DI)
	MOVOU X13, 208(DI)
	MOVOU X14, 224(DI)
	MOVOU X15, 240(DI)
	LEAQ  64(SI), SI
	SUBQ  $0x40, BX
	MOVQ  $0x00000006, CX
	MOVQ  $0x00000004, R9
	CMPQ  BX, $0xc0
	JG    sealSSEMainLoop
	MOVQ  BX, CX
	TESTQ BX, BX
	JE    sealSSE128SealHash
	MOVQ  $0x00000006, CX
	CMPQ  BX, $0x40
	JBE   sealSSETail64
	CMPQ  BX, $0x80
	JBE   sealSSETail128
	JMP   sealSSETail192

sealSSETail64:
	MOVO  ·chacha20Constants<>+0(SB), X1
	MOVO  32(BP), X4
	MOVO  48(BP), X7
	MOVO  128(BP), X10
	PADDL ·sseIncMask<>+0(SB), X10
	MOVO  X10, 80(BP)

sealSSETail64LoopA:
	ADDQ  (DI), R10
	ADCQ  8(DI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(DI), DI

sealSSETail64LoopB:
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X13)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X13
	PSLLL $0x0c, X13
	PSRLL $0x14, X4
	PXOR  X13, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X13)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X13
	PSLLL $0x07, X13
	PSRLL $0x19, X4
	PXOR  X13, X4
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x0c
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X13)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X13
	PSLLL $0x0c, X13
	PSRLL $0x14, X4
	PXOR  X13, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X13)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X13
	PSLLL $0x07, X13
	PSRLL $0x19, X4
	PXOR  X13, X4
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x04
	ADDQ  (DI), R10
	ADCQ  8(DI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(DI), DI
	DECQ  CX
	JG    sealSSETail64LoopA
	DECQ  R9
	JGE   sealSSETail64LoopB
	PADDL ·chacha20Constants<>+0(SB), X1
	PADDL 32(BP), X4
	PADDL 48(BP), X7
	PADDL 80(BP), X10
	JMP   sealSSE128Seal

sealSSETail128:
	MOVO  ·chacha20Constants<>+0(SB), X0
	MOVO  32(BP), X3
	MOVO  48(BP), X6
	MOVO  128(BP), X9
	PADDL ·sseIncMask<>+0(SB), X9
	MOVO  X9, 80(BP)
	MOVO  X0, X1
	MOVO  X3, X4
	MOVO  X6, X7
	MOVO  X9, X10
	PADDL ·sseIncMask<>+0(SB), X10
	MOVO  X10, 96(BP)

sealSSETail128LoopA:
	ADDQ  (DI), R10
	ADCQ  8(DI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(DI), DI

sealSSETail128LoopB:
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X3
	PXOR  X12, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X3
	PXOR  X12, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X4
	PXOR  X12, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X4
	PXOR  X12, X4
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x0c
	ADDQ  (DI), R10
	ADCQ  8(DI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(DI), DI
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X3
	PXOR  X12, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X3
	PXOR  X12, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X4
	PXOR  X12, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X4
	PXOR  X12, X4
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x04
	DECQ  CX
	JG    sealSSETail128LoopA
	DECQ  R9
	JGE   sealSSETail128LoopB
	PADDL ·chacha20Constants<>+0(SB), X0
	PADDL ·chacha20Constants<>+0(SB), X1
	PADDL 32(BP), X3
	PADDL 32(BP), X4
	PADDL 48(BP), X6
	PADDL 48(BP), X7
	PADDL 80(BP), X9
	PADDL 96(BP), X10
	MOVOU (SI), X12
	MOVOU 16(SI), X13
	MOVOU 32(SI), X14
	MOVOU 48(SI), X15
	PXOR  X12, X0
	PXOR  X13, X3
	PXOR  X14, X6
	PXOR  X15, X9
	MOVOU X0, (DI)
	MOVOU X3, 16(DI)
	MOVOU X6, 32(DI)
	MOVOU X9, 48(DI)
	MOVQ  $0x00000040, CX
	LEAQ  64(SI), SI
	SUBQ  $0x40, BX
	JMP   sealSSE128SealHash

sealSSETail192:
	MOVO  ·chacha20Constants<>+0(SB), X0
	MOVO  32(BP), X3
	MOVO  48(BP), X6
	MOVO  128(BP), X9
	PADDL ·sseIncMask<>+0(SB), X9
	MOVO  X9, 80(BP)
	MOVO  X0, X1
	MOVO  X3, X4
	MOVO  X6, X7
	MOVO  X9, X10
	PADDL ·sseIncMask<>+0(SB), X10
	MOVO  X10, 96(BP)
	MOVO  X1, X2
	MOVO  X4, X5
	MOVO  X7, X8
	MOVO  X10, X11
	PADDL ·sseIncMask<>+0(SB), X11
	MOVO  X11, 112(BP)

sealSSETail192LoopA:
	ADDQ  (DI), R10
	ADCQ  8(DI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(DI), DI

sealSSETail192LoopB:
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X3
	PXOR  X12, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X3
	PXOR  X12, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X4
	PXOR  X12, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X4
	PXOR  X12, X4
	PADDD X5, X2
	PXOR  X2, X11
	ROL16(X11, X12)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X5
	PXOR  X12, X5
	PADDD X5, X2
	PXOR  X2, X11
	ROL8(X11, X12)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X5
	PXOR  X12, X5
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc0
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	ADDQ  (DI), R10
	ADCQ  8(DI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(DI), DI
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X3
	PXOR  X12, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X3
	PXOR  X12, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X4
	PXOR  X12, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X4
	PXOR  X12, X4
	PADDD X5, X2
	PXOR  X2, X11
	ROL16(X11, X12)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X5
	PXOR  X12, X5
	PADDD X5, X2
	PXOR  X2, X11
	ROL8(X11, X12)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X5
	PXOR  X12, X5
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc0
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	DECQ  CX
	JG    sealSSETail192LoopA
	DECQ  R9
	JGE   sealSSETail192LoopB
	PADDL ·chacha20Constants<>+0(SB), X0
	PADDL ·chacha20Constants<>+0(SB), X1
	PADDL ·chacha20Constants<>+0(SB), X2
	PADDL 32(BP), X3
	PADDL 32(BP), X4
	PADDL 32(BP), X5
	PADDL 48(BP), X6
	PADDL 48(BP), X7
	PADDL 48(BP), X8
	PADDL 80(BP), X9
	PADDL 96(BP), X10
	PADDL 112(BP), X11
	MOVOU (SI), X12
	MOVOU 16(SI), X13
	MOVOU 32(SI), X14
	MOVOU 48(SI), X15
	PXOR  X12, X0
	PXOR  X13, X3
	PXOR  X14, X6
	PXOR  X15, X9
	MOVOU X0, (DI)
	MOVOU X3, 16(DI)
	MOVOU X6, 32(DI)
	MOVOU X9, 48(DI)
	MOVOU 64(SI), X12
	MOVOU 80(SI), X13
	MOVOU 96(SI), X14
	MOVOU 112(SI), X15
	PXOR  X12, X1
	PXOR  X13, X4
	PXOR  X14, X7
	PXOR  X15, X10
	MOVOU X1, 64(DI)
	MOVOU X4, 80(DI)
	MOVOU X7, 96(DI)
	MOVOU X10, 112(DI)
	MOVO  X2, X1
	MOVO  X5, X4
	MOVO  X8, X7
	MOVO  X11, X10
	MOVQ  $0x00000080, CX
	LEAQ  128(SI), SI
	SUBQ  $0x80, BX
	JMP   sealSSE128SealHash

sealSSE128:
	MOVOU ·chacha20Constants<>+0(SB), X0
	MOVOU 16(R8), X3
	MOVOU 32(R8), X6
	MOVOU 48(R8), X9
	MOVO  X0, X1
	MOVO  X3, X4
	MOVO  X6, X7
	MOVO  X9, X10
	PADDL ·sseIncMask<>+0(SB), X10
	MOVO  X1, X2
	MOVO  X4, X5
	MOVO  X7, X8
	MOVO  X10, X11
	PADDL ·sseIncMask<>+0(SB), X11
	MOVO  X3, X13
	MOVO  X6, X14
	MOVO  X10, X15
	MOVQ  $0x0000000a, R9

sealSSE128InnerCipherLoop:
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X3
	PXOR  X12, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X3
	PXOR  X12, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X4
	PXOR  X12, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X4
	PXOR  X12, X4
	PADDD X5, X2
	PXOR  X2, X11
	ROL16(X11, X12)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X5
	PXOR  X12, X5
	PADDD X5, X2
	PXOR  X2, X11
	ROL8(X11, X12)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X5
	PXOR  X12, X5
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc0
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	PADDD X3, X0
	PXOR  X0, X9
	ROL16(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X3
	PXOR  X12, X3
	PADDD X3, X0
	PXOR  X0, X9
	ROL8(X9, X12)
	PADDD X9, X6
	PXOR  X6, X3
	MOVO  X3, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X3
	PXOR  X12, X3
	PADDD X4, X1
	PXOR  X1, X10
	ROL16(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X4
	PXOR  X12, X4
	PADDD X4, X1
	PXOR  X1, X10
	ROL8(X10, X12)
	PADDD X10, X7
	PXOR  X7, X4
	MOVO  X4, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X4
	PXOR  X12, X4
	PADDD X5, X2
	PXOR  X2, X11
	ROL16(X11, X12)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X12
	PSLLL $0x0c, X12
	PSRLL $0x14, X5
	PXOR  X12, X5
	PADDD X5, X2
	PXOR  X2, X11
	ROL8(X11, X12)
	PADDD X11, X8
	PXOR  X8, X5
	MOVO  X5, X12
	PSLLL $0x07, X12
	PSRLL $0x19, X5
	PXOR  X12, X5
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xe4
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xed
	BYTE  $0x0c
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xf6
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xff
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc0
	BYTE  $0x08
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xc9
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xd2
	BYTE  $0x04
	BYTE  $0x66
	BYTE  $0x45
	BYTE  $0x0f
	BYTE  $0x3a
	BYTE  $0x0f
	BYTE  $0xdb
	BYTE  $0x04
	DECQ  R9
	JNE   sealSSE128InnerCipherLoop

	// A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded
	PADDL ·chacha20Constants<>+0(SB), X0
	PADDL ·chacha20Constants<>+0(SB), X1
	PADDL ·chacha20Constants<>+0(SB), X2
	PADDL X13, X3
	PADDL X13, X4
	PADDL X13, X5
	PADDL X14, X7
	PADDL X14, X8
	PADDL X15, X10
	PADDL ·sseIncMask<>+0(SB), X15
	PADDL X15, X11
	PAND  ·polyClampMask<>+0(SB), X0
	MOVOU X0, (BP)
	MOVOU X3, 16(BP)

	// Hash
	MOVQ ad_len+80(FP), R9
	CALL polyHashADInternal<>(SB)
	XORQ CX, CX

sealSSE128SealHash:
	CMPQ  CX, $0x10
	JB    sealSSE128Seal
	ADDQ  (DI), R10
	ADCQ  8(DI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	SUBQ  $0x10, CX
	ADDQ  $0x10, DI
	JMP   sealSSE128SealHash

sealSSE128Seal:
	CMPQ BX, $0x10
	JB   sealSSETail
	SUBQ $0x10, BX

	// Load for decryption
	MOVOU (SI), X12
	PXOR  X12, X1
	MOVOU X1, (DI)
	LEAQ  16(SI), SI
	LEAQ  16(DI), DI

	// Extract for hashing
	MOVQ   X1, R13
	PSRLDQ $0x08, X1
	MOVQ   X1, R14
	ADDQ   R13, R10
	ADCQ   R14, R11
	ADCQ   $0x01, R12
	MOVQ   (BP), AX
	MOVQ   AX, R15
	MULQ   R10
	MOVQ   AX, R13
	MOVQ   DX, R14
	MOVQ   (BP), AX
	MULQ   R11
	IMULQ  R12, R15
	ADDQ   AX, R14
	ADCQ   DX, R15
	MOVQ   8(BP), AX
	MOVQ   AX, R8
	MULQ   R10
	ADDQ   AX, R14
	ADCQ   $0x00, DX
	MOVQ   DX, R10
	MOVQ   8(BP), AX
	MULQ   R11
	ADDQ   AX, R15
	ADCQ   $0x00, DX
	IMULQ  R12, R8
	ADDQ   R10, R15
	ADCQ   DX, R8
	MOVQ   R13, R10
	MOVQ   R14, R11
	MOVQ   R15, R12
	ANDQ   $0x03, R12
	MOVQ   R15, R13
	ANDQ   $-4, R13
	MOVQ   R8, R14
	SHRQ   $0x02, R8, R15
	SHRQ   $0x02, R8
	ADDQ   R13, R10
	ADCQ   R14, R11
	ADCQ   $0x00, R12
	ADDQ   R15, R10
	ADCQ   R8, R11
	ADCQ   $0x00, R12

	// Shift the stream "left"
	MOVO X4, X1
	MOVO X7, X4
	MOVO X10, X7
	MOVO X2, X10
	MOVO X5, X2
	MOVO X8, X5
	MOVO X11, X8
	JMP  sealSSE128Seal

sealSSETail:
	TESTQ BX, BX
	JE    sealSSEFinalize

	// We can only load the PT one byte at a time to avoid read after end of buffer
	MOVQ BX, R9
	SHLQ $0x04, R9
	LEAQ ·andMask<>+0(SB), R13
	MOVQ BX, CX
	LEAQ -1(SI)(BX*1), SI
	XORQ R15, R15
	XORQ R8, R8
	XORQ AX, AX

sealSSETailLoadLoop:
	SHLQ   $0x08, R15, R8
	SHLQ   $0x08, R15
	MOVB   (SI), AX
	XORQ   AX, R15
	LEAQ   -1(SI), SI
	DECQ   CX
	JNE    sealSSETailLoadLoop
	MOVQ   R15, 64(BP)
	MOVQ   R8, 72(BP)
	PXOR   64(BP), X1
	MOVOU  X1, (DI)
	MOVOU  -16(R13)(R9*1), X12
	PAND   X12, X1
	MOVQ   X1, R13
	PSRLDQ $0x08, X1
	MOVQ   X1, R14
	ADDQ   R13, R10
	ADCQ   R14, R11
	ADCQ   $0x01, R12
	MOVQ   (BP), AX
	MOVQ   AX, R15
	MULQ   R10
	MOVQ   AX, R13
	MOVQ   DX, R14
	MOVQ   (BP), AX
	MULQ   R11
	IMULQ  R12, R15
	ADDQ   AX, R14
	ADCQ   DX, R15
	MOVQ   8(BP), AX
	MOVQ   AX, R8
	MULQ   R10
	ADDQ   AX, R14
	ADCQ   $0x00, DX
	MOVQ   DX, R10
	MOVQ   8(BP), AX
	MULQ   R11
	ADDQ   AX, R15
	ADCQ   $0x00, DX
	IMULQ  R12, R8
	ADDQ   R10, R15
	ADCQ   DX, R8
	MOVQ   R13, R10
	MOVQ   R14, R11
	MOVQ   R15, R12
	ANDQ   $0x03, R12
	MOVQ   R15, R13
	ANDQ   $-4, R13
	MOVQ   R8, R14
	SHRQ   $0x02, R8, R15
	SHRQ   $0x02, R8
	ADDQ   R13, R10
	ADCQ   R14, R11
	ADCQ   $0x00, R12
	ADDQ   R15, R10
	ADCQ   R8, R11
	ADCQ   $0x00, R12
	ADDQ   BX, DI

sealSSEFinalize:
	// Hash in the buffer lengths
	ADDQ  ad_len+80(FP), R10
	ADCQ  src_len+56(FP), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12

	// Final reduce
	MOVQ    R10, R13
	MOVQ    R11, R14
	MOVQ    R12, R15
	SUBQ    $-5, R10
	SBBQ    $-1, R11
	SBBQ    $0x03, R12
	CMOVQCS R13, R10
	CMOVQCS R14, R11
	CMOVQCS R15, R12

	// Add in the "s" part of the key
	ADDQ 16(BP), R10
	ADCQ 24(BP), R11

	// Finally store the tag at the end of the message
	MOVQ R10, (DI)
	MOVQ R11, 8(DI)
	RET

chacha20Poly1305Seal_AVX2:
	VZEROUPPER
	VMOVDQU ·chacha20Constants<>+0(SB), Y0
	BYTE    $0xc4
	BYTE    $0x42
	BYTE    $0x7d
	BYTE    $0x5a
	BYTE    $0x70
	BYTE    $0x10
	BYTE    $0xc4
	BYTE    $0x42
	BYTE    $0x7d
	BYTE    $0x5a
	BYTE    $0x60
	BYTE    $0x20
	BYTE    $0xc4
	BYTE    $0xc2
	BYTE    $0x7d
	BYTE    $0x5a
	BYTE    $0x60
	BYTE    $0x30
	VPADDD  ·avx2InitMask<>+0(SB), Y4, Y4

	// Special optimizations, for very short buffers
	CMPQ BX, $0x000000c0
	JBE  seal192AVX2
	CMPQ BX, $0x00000140
	JBE  seal320AVX2

	// For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream
	VMOVDQA Y0, Y5
	VMOVDQA Y0, Y6
	VMOVDQA Y0, Y7
	VMOVDQA Y14, Y9
	VMOVDQA Y14, Y10
	VMOVDQA Y14, Y11
	VMOVDQA Y14, 32(BP)
	VMOVDQA Y12, Y13
	VMOVDQA Y12, Y8
	VMOVDQA Y12, Y15
	VMOVDQA Y12, 64(BP)
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
	VMOVDQA Y4, 96(BP)
	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
	VMOVDQA Y1, 128(BP)
	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
	VMOVDQA Y2, 160(BP)
	VMOVDQA Y3, 192(BP)
	MOVQ    $0x0000000a, R9

sealAVX2IntroLoop:
	VMOVDQA    Y15, 224(BP)
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x0c, Y14, Y15
	VPSRLD     $0x14, Y14, Y14
	VPXOR      Y15, Y14, Y14
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x07, Y14, Y15
	VPSRLD     $0x19, Y14, Y14
	VPXOR      Y15, Y14, Y14
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x0c, Y9, Y15
	VPSRLD     $0x14, Y9, Y9
	VPXOR      Y15, Y9, Y9
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x07, Y9, Y15
	VPSRLD     $0x19, Y9, Y9
	VPXOR      Y15, Y9, Y9
	VPADDD     Y10, Y6, Y6
	VPXOR      Y6, Y2, Y2
	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
	VPADDD     Y2, Y8, Y8
	VPXOR      Y8, Y10, Y10
	VPSLLD     $0x0c, Y10, Y15
	VPSRLD     $0x14, Y10, Y10
	VPXOR      Y15, Y10, Y10
	VPADDD     Y10, Y6, Y6
	VPXOR      Y6, Y2, Y2
	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
	VPADDD     Y2, Y8, Y8
	VPXOR      Y8, Y10, Y10
	VPSLLD     $0x07, Y10, Y15
	VPSRLD     $0x19, Y10, Y10
	VPXOR      Y15, Y10, Y10
	VMOVDQA    224(BP), Y15
	VMOVDQA    Y13, 224(BP)
	VPADDD     Y11, Y7, Y7
	VPXOR      Y7, Y3, Y3
	VPSHUFB    ·rol16<>+0(SB), Y3, Y3
	VPADDD     Y3, Y15, Y15
	VPXOR      Y15, Y11, Y11
	VPSLLD     $0x0c, Y11, Y13
	VPSRLD     $0x14, Y11, Y11
	VPXOR      Y13, Y11, Y11
	VPADDD     Y11, Y7, Y7
	VPXOR      Y7, Y3, Y3
	VPSHUFB    ·rol8<>+0(SB), Y3, Y3
	VPADDD     Y3, Y15, Y15
	VPXOR      Y15, Y11, Y11
	VPSLLD     $0x07, Y11, Y13
	VPSRLD     $0x19, Y11, Y11
	VPXOR      Y13, Y11, Y11
	VMOVDQA    224(BP), Y13
	VPALIGNR   $0x04, Y14, Y14, Y14
	VPALIGNR   $0x08, Y12, Y12, Y12
	VPALIGNR   $0x0c, Y4, Y4, Y4
	VPALIGNR   $0x04, Y9, Y9, Y9
	VPALIGNR   $0x08, Y13, Y13, Y13
	VPALIGNR   $0x0c, Y1, Y1, Y1
	VPALIGNR   $0x04, Y10, Y10, Y10
	VPALIGNR   $0x08, Y8, Y8, Y8
	VPALIGNR   $0x0c, Y2, Y2, Y2
	VPALIGNR   $0x04, Y11, Y11, Y11
	VPALIGNR   $0x08, Y15, Y15, Y15
	VPALIGNR   $0x0c, Y3, Y3, Y3
	VMOVDQA    Y15, 224(BP)
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x0c, Y14, Y15
	VPSRLD     $0x14, Y14, Y14
	VPXOR      Y15, Y14, Y14
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x07, Y14, Y15
	VPSRLD     $0x19, Y14, Y14
	VPXOR      Y15, Y14, Y14
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x0c, Y9, Y15
	VPSRLD     $0x14, Y9, Y9
	VPXOR      Y15, Y9, Y9
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x07, Y9, Y15
	VPSRLD     $0x19, Y9, Y9
	VPXOR      Y15, Y9, Y9
	VPADDD     Y10, Y6, Y6
	VPXOR      Y6, Y2, Y2
	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
	VPADDD     Y2, Y8, Y8
	VPXOR      Y8, Y10, Y10
	VPSLLD     $0x0c, Y10, Y15
	VPSRLD     $0x14, Y10, Y10
	VPXOR      Y15, Y10, Y10
	VPADDD     Y10, Y6, Y6
	VPXOR      Y6, Y2, Y2
	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
	VPADDD     Y2, Y8, Y8
	VPXOR      Y8, Y10, Y10
	VPSLLD     $0x07, Y10, Y15
	VPSRLD     $0x19, Y10, Y10
	VPXOR      Y15, Y10, Y10
	VMOVDQA    224(BP), Y15
	VMOVDQA    Y13, 224(BP)
	VPADDD     Y11, Y7, Y7
	VPXOR      Y7, Y3, Y3
	VPSHUFB    ·rol16<>+0(SB), Y3, Y3
	VPADDD     Y3, Y15, Y15
	VPXOR      Y15, Y11, Y11
	VPSLLD     $0x0c, Y11, Y13
	VPSRLD     $0x14, Y11, Y11
	VPXOR      Y13, Y11, Y11
	VPADDD     Y11, Y7, Y7
	VPXOR      Y7, Y3, Y3
	VPSHUFB    ·rol8<>+0(SB), Y3, Y3
	VPADDD     Y3, Y15, Y15
	VPXOR      Y15, Y11, Y11
	VPSLLD     $0x07, Y11, Y13
	VPSRLD     $0x19, Y11, Y11
	VPXOR      Y13, Y11, Y11
	VMOVDQA    224(BP), Y13
	VPALIGNR   $0x0c, Y14, Y14, Y14
	VPALIGNR   $0x08, Y12, Y12, Y12
	VPALIGNR   $0x04, Y4, Y4, Y4
	VPALIGNR   $0x0c, Y9, Y9, Y9
	VPALIGNR   $0x08, Y13, Y13, Y13
	VPALIGNR   $0x04, Y1, Y1, Y1
	VPALIGNR   $0x0c, Y10, Y10, Y10
	VPALIGNR   $0x08, Y8, Y8, Y8
	VPALIGNR   $0x04, Y2, Y2, Y2
	VPALIGNR   $0x0c, Y11, Y11, Y11
	VPALIGNR   $0x08, Y15, Y15, Y15
	VPALIGNR   $0x04, Y3, Y3, Y3
	DECQ       R9
	JNE        sealAVX2IntroLoop
	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
	VPADDD     ·chacha20Constants<>+0(SB), Y7, Y7
	VPADDD     32(BP), Y14, Y14
	VPADDD     32(BP), Y9, Y9
	VPADDD     32(BP), Y10, Y10
	VPADDD     32(BP), Y11, Y11
	VPADDD     64(BP), Y12, Y12
	VPADDD     64(BP), Y13, Y13
	VPADDD     64(BP), Y8, Y8
	VPADDD     64(BP), Y15, Y15
	VPADDD     96(BP), Y4, Y4
	VPADDD     128(BP), Y1, Y1
	VPADDD     160(BP), Y2, Y2
	VPADDD     192(BP), Y3, Y3
	VPERM2I128 $0x13, Y12, Y4, Y12
	VPERM2I128 $0x02, Y0, Y14, Y4
	VPERM2I128 $0x13, Y0, Y14, Y0

	// Clamp and store poly key
	VPAND   ·polyClampMask<>+0(SB), Y4, Y4
	VMOVDQA Y4, (BP)

	// Hash AD
	MOVQ ad_len+80(FP), R9
	CALL polyHashADInternal<>(SB)

	// Can store at least 320 bytes
	VPXOR      (SI), Y0, Y0
	VPXOR      32(SI), Y12, Y12
	VMOVDQU    Y0, (DI)
	VMOVDQU    Y12, 32(DI)
	VPERM2I128 $0x02, Y5, Y9, Y0
	VPERM2I128 $0x02, Y13, Y1, Y14
	VPERM2I128 $0x13, Y5, Y9, Y12
	VPERM2I128 $0x13, Y13, Y1, Y4
	VPXOR      64(SI), Y0, Y0
	VPXOR      96(SI), Y14, Y14
	VPXOR      128(SI), Y12, Y12
	VPXOR      160(SI), Y4, Y4
	VMOVDQU    Y0, 64(DI)
	VMOVDQU    Y14, 96(DI)
	VMOVDQU    Y12, 128(DI)
	VMOVDQU    Y4, 160(DI)
	VPERM2I128 $0x02, Y6, Y10, Y0
	VPERM2I128 $0x02, Y8, Y2, Y14
	VPERM2I128 $0x13, Y6, Y10, Y12
	VPERM2I128 $0x13, Y8, Y2, Y4
	VPXOR      192(SI), Y0, Y0
	VPXOR      224(SI), Y14, Y14
	VPXOR      256(SI), Y12, Y12
	VPXOR      288(SI), Y4, Y4
	VMOVDQU    Y0, 192(DI)
	VMOVDQU    Y14, 224(DI)
	VMOVDQU    Y12, 256(DI)
	VMOVDQU    Y4, 288(DI)
	MOVQ       $0x00000140, CX
	SUBQ       $0x00000140, BX
	LEAQ       320(SI), SI
	VPERM2I128 $0x02, Y7, Y11, Y0
	VPERM2I128 $0x02, Y15, Y3, Y14
	VPERM2I128 $0x13, Y7, Y11, Y12
	VPERM2I128 $0x13, Y15, Y3, Y4
	CMPQ       BX, $0x80
	JBE        sealAVX2SealHash
	VPXOR      (SI), Y0, Y0
	VPXOR      32(SI), Y14, Y14
	VPXOR      64(SI), Y12, Y12
	VPXOR      96(SI), Y4, Y4
	VMOVDQU    Y0, 320(DI)
	VMOVDQU    Y14, 352(DI)
	VMOVDQU    Y12, 384(DI)
	VMOVDQU    Y4, 416(DI)
	SUBQ       $0x80, BX
	LEAQ       128(SI), SI
	MOVQ       $0x00000008, CX
	MOVQ       $0x00000002, R9
	CMPQ       BX, $0x80
	JBE        sealAVX2Tail128
	CMPQ       BX, $0x00000100
	JBE        sealAVX2Tail256
	CMPQ       BX, $0x00000180
	JBE        sealAVX2Tail384
	CMPQ       BX, $0x00000200
	JBE        sealAVX2Tail512

	// We have 448 bytes to hash, but main loop hashes 512 bytes at a time - perform some rounds, before the main loop
	VMOVDQA  ·chacha20Constants<>+0(SB), Y0
	VMOVDQA  Y0, Y5
	VMOVDQA  Y0, Y6
	VMOVDQA  Y0, Y7
	VMOVDQA  32(BP), Y14
	VMOVDQA  Y14, Y9
	VMOVDQA  Y14, Y10
	VMOVDQA  Y14, Y11
	VMOVDQA  64(BP), Y12
	VMOVDQA  Y12, Y13
	VMOVDQA  Y12, Y8
	VMOVDQA  Y12, Y15
	VMOVDQA  192(BP), Y4
	VPADDD   ·avx2IncMask<>+0(SB), Y4, Y4
	VPADDD   ·avx2IncMask<>+0(SB), Y4, Y1
	VPADDD   ·avx2IncMask<>+0(SB), Y1, Y2
	VPADDD   ·avx2IncMask<>+0(SB), Y2, Y3
	VMOVDQA  Y4, 96(BP)
	VMOVDQA  Y1, 128(BP)
	VMOVDQA  Y2, 160(BP)
	VMOVDQA  Y3, 192(BP)
	VMOVDQA  Y15, 224(BP)
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x0c, Y14, Y15
	VPSRLD   $0x14, Y14, Y14
	VPXOR    Y15, Y14, Y14
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x07, Y14, Y15
	VPSRLD   $0x19, Y14, Y14
	VPXOR    Y15, Y14, Y14
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x0c, Y9, Y15
	VPSRLD   $0x14, Y9, Y9
	VPXOR    Y15, Y9, Y9
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x07, Y9, Y15
	VPSRLD   $0x19, Y9, Y9
	VPXOR    Y15, Y9, Y9
	VPADDD   Y10, Y6, Y6
	VPXOR    Y6, Y2, Y2
	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
	VPADDD   Y2, Y8, Y8
	VPXOR    Y8, Y10, Y10
	VPSLLD   $0x0c, Y10, Y15
	VPSRLD   $0x14, Y10, Y10
	VPXOR    Y15, Y10, Y10
	VPADDD   Y10, Y6, Y6
	VPXOR    Y6, Y2, Y2
	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
	VPADDD   Y2, Y8, Y8
	VPXOR    Y8, Y10, Y10
	VPSLLD   $0x07, Y10, Y15
	VPSRLD   $0x19, Y10, Y10
	VPXOR    Y15, Y10, Y10
	VMOVDQA  224(BP), Y15
	VMOVDQA  Y13, 224(BP)
	VPADDD   Y11, Y7, Y7
	VPXOR    Y7, Y3, Y3
	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
	VPADDD   Y3, Y15, Y15
	VPXOR    Y15, Y11, Y11
	VPSLLD   $0x0c, Y11, Y13
	VPSRLD   $0x14, Y11, Y11
	VPXOR    Y13, Y11, Y11
	VPADDD   Y11, Y7, Y7
	VPXOR    Y7, Y3, Y3
	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
	VPADDD   Y3, Y15, Y15
	VPXOR    Y15, Y11, Y11
	VPSLLD   $0x07, Y11, Y13
	VPSRLD   $0x19, Y11, Y11
	VPXOR    Y13, Y11, Y11
	VMOVDQA  224(BP), Y13
	VPALIGNR $0x04, Y14, Y14, Y14
	VPALIGNR $0x08, Y12, Y12, Y12
	VPALIGNR $0x0c, Y4, Y4, Y4
	VPALIGNR $0x04, Y9, Y9, Y9
	VPALIGNR $0x08, Y13, Y13, Y13
	VPALIGNR $0x0c, Y1, Y1, Y1
	VPALIGNR $0x04, Y10, Y10, Y10
	VPALIGNR $0x08, Y8, Y8, Y8
	VPALIGNR $0x0c, Y2, Y2, Y2
	VPALIGNR $0x04, Y11, Y11, Y11
	VPALIGNR $0x08, Y15, Y15, Y15
	VPALIGNR $0x0c, Y3, Y3, Y3
	VMOVDQA  Y15, 224(BP)
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x0c, Y14, Y15
	VPSRLD   $0x14, Y14, Y14
	VPXOR    Y15, Y14, Y14
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x07, Y14, Y15
	VPSRLD   $0x19, Y14, Y14
	VPXOR    Y15, Y14, Y14
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x0c, Y9, Y15
	VPSRLD   $0x14, Y9, Y9
	VPXOR    Y15, Y9, Y9
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x07, Y9, Y15
	VPSRLD   $0x19, Y9, Y9
	VPXOR    Y15, Y9, Y9
	VPADDD   Y10, Y6, Y6
	VPXOR    Y6, Y2, Y2
	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
	VPADDD   Y2, Y8, Y8
	VPXOR    Y8, Y10, Y10
	VPSLLD   $0x0c, Y10, Y15
	VPSRLD   $0x14, Y10, Y10
	VPXOR    Y15, Y10, Y10
	VPADDD   Y10, Y6, Y6
	VPXOR    Y6, Y2, Y2
	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
	VPADDD   Y2, Y8, Y8
	VPXOR    Y8, Y10, Y10
	VPSLLD   $0x07, Y10, Y15
	VPSRLD   $0x19, Y10, Y10
	VPXOR    Y15, Y10, Y10
	VMOVDQA  224(BP), Y15
	VMOVDQA  Y13, 224(BP)
	VPADDD   Y11, Y7, Y7
	VPXOR    Y7, Y3, Y3
	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
	VPADDD   Y3, Y15, Y15
	VPXOR    Y15, Y11, Y11
	VPSLLD   $0x0c, Y11, Y13
	VPSRLD   $0x14, Y11, Y11
	VPXOR    Y13, Y11, Y11
	VPADDD   Y11, Y7, Y7
	VPXOR    Y7, Y3, Y3
	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
	VPADDD   Y3, Y15, Y15
	VPXOR    Y15, Y11, Y11
	VPSLLD   $0x07, Y11, Y13
	VPSRLD   $0x19, Y11, Y11
	VPXOR    Y13, Y11, Y11
	VMOVDQA  224(BP), Y13
	VPALIGNR $0x0c, Y14, Y14, Y14
	VPALIGNR $0x08, Y12, Y12, Y12
	VPALIGNR $0x04, Y4, Y4, Y4
	VPALIGNR $0x0c, Y9, Y9, Y9
	VPALIGNR $0x08, Y13, Y13, Y13
	VPALIGNR $0x04, Y1, Y1, Y1
	VPALIGNR $0x0c, Y10, Y10, Y10
	VPALIGNR $0x08, Y8, Y8, Y8
	VPALIGNR $0x04, Y2, Y2, Y2
	VPALIGNR $0x0c, Y11, Y11, Y11
	VPALIGNR $0x08, Y15, Y15, Y15
	VPALIGNR $0x04, Y3, Y3, Y3
	VPADDD   Y14, Y0, Y0
	VPADDD   Y9, Y5, Y5
	VPADDD   Y10, Y6, Y6
	VPADDD   Y11, Y7, Y7
	VPXOR    Y0, Y4, Y4
	VPXOR    Y5, Y1, Y1
	VPXOR    Y6, Y2, Y2
	VPXOR    Y7, Y3, Y3
	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
	VPADDD   Y4, Y12, Y12
	VPADDD   Y1, Y13, Y13
	VPADDD   Y2, Y8, Y8
	VPADDD   Y3, Y15, Y15
	VPXOR    Y12, Y14, Y14
	VPXOR    Y13, Y9, Y9
	VPXOR    Y8, Y10, Y10
	VPXOR    Y15, Y11, Y11
	VMOVDQA  Y15, 224(BP)
	VPSLLD   $0x0c, Y14, Y15
	VPSRLD   $0x14, Y14, Y14
	VPXOR    Y15, Y14, Y14
	VPSLLD   $0x0c, Y9, Y15
	VPSRLD   $0x14, Y9, Y9
	VPXOR    Y15, Y9, Y9
	VPSLLD   $0x0c, Y10, Y15
	VPSRLD   $0x14, Y10, Y10
	VPXOR    Y15, Y10, Y10
	VPSLLD   $0x0c, Y11, Y15
	VPSRLD   $0x14, Y11, Y11
	VPXOR    Y15, Y11, Y11
	VMOVDQA  224(BP), Y15
	SUBQ     $0x10, DI
	MOVQ     $0x00000009, CX
	JMP      sealAVX2InternalLoopStart

sealAVX2MainLoop:
	VMOVDQU ·chacha20Constants<>+0(SB), Y0
	VMOVDQA Y0, Y5
	VMOVDQA Y0, Y6
	VMOVDQA Y0, Y7
	VMOVDQA 32(BP), Y14
	VMOVDQA Y14, Y9
	VMOVDQA Y14, Y10
	VMOVDQA Y14, Y11
	VMOVDQA 64(BP), Y12
	VMOVDQA Y12, Y13
	VMOVDQA Y12, Y8
	VMOVDQA Y12, Y15
	VMOVDQA 192(BP), Y4
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
	VMOVDQA Y4, 96(BP)
	VMOVDQA Y1, 128(BP)
	VMOVDQA Y2, 160(BP)
	VMOVDQA Y3, 192(BP)
	MOVQ    $0x0000000a, CX

sealAVX2InternalLoop:
	ADDQ    (DI), R10
	ADCQ    8(DI), R11
	ADCQ    $0x01, R12
	VPADDD  Y14, Y0, Y0
	VPADDD  Y9, Y5, Y5
	VPADDD  Y10, Y6, Y6
	VPADDD  Y11, Y7, Y7
	MOVQ    (BP), DX
	MOVQ    DX, R15
	MULXQ   R10, R13, R14
	IMULQ   R12, R15
	MULXQ   R11, AX, DX
	ADDQ    AX, R14
	ADCQ    DX, R15
	VPXOR   Y0, Y4, Y4
	VPXOR   Y5, Y1, Y1
	VPXOR   Y6, Y2, Y2
	VPXOR   Y7, Y3, Y3
	VPSHUFB ·rol16<>+0(SB), Y4, Y4
	VPSHUFB ·rol16<>+0(SB), Y1, Y1
	VPSHUFB ·rol16<>+0(SB), Y2, Y2
	VPSHUFB ·rol16<>+0(SB), Y3, Y3
	MOVQ    8(BP), DX
	MULXQ   R10, R10, AX
	ADDQ    R10, R14
	MULXQ   R11, R11, R8
	ADCQ    R11, R15
	ADCQ    $0x00, R8
	VPADDD  Y4, Y12, Y12
	VPADDD  Y1, Y13, Y13
	VPADDD  Y2, Y8, Y8
	VPADDD  Y3, Y15, Y15
	VPXOR   Y12, Y14, Y14
	VPXOR   Y13, Y9, Y9
	VPXOR   Y8, Y10, Y10
	VPXOR   Y15, Y11, Y11
	IMULQ   R12, DX
	ADDQ    AX, R15
	ADCQ    DX, R8
	VMOVDQA Y15, 224(BP)
	VPSLLD  $0x0c, Y14, Y15
	VPSRLD  $0x14, Y14, Y14
	VPXOR   Y15, Y14, Y14
	VPSLLD  $0x0c, Y9, Y15
	VPSRLD  $0x14, Y9, Y9
	VPXOR   Y15, Y9, Y9
	VPSLLD  $0x0c, Y10, Y15
	VPSRLD  $0x14, Y10, Y10
	VPXOR   Y15, Y10, Y10
	VPSLLD  $0x0c, Y11, Y15
	VPSRLD  $0x14, Y11, Y11
	VPXOR   Y15, Y11, Y11
	VMOVDQA 224(BP), Y15
	MOVQ    R13, R10
	MOVQ    R14, R11
	MOVQ    R15, R12
	ANDQ    $0x03, R12
	MOVQ    R15, R13
	ANDQ    $-4, R13
	MOVQ    R8, R14
	SHRQ    $0x02, R8, R15
	SHRQ    $0x02, R8
	ADDQ    R13, R10
	ADCQ    R14, R11
	ADCQ    $0x00, R12
	ADDQ    R15, R10
	ADCQ    R8, R11
	ADCQ    $0x00, R12

sealAVX2InternalLoopStart:
	VPADDD   Y14, Y0, Y0
	VPADDD   Y9, Y5, Y5
	VPADDD   Y10, Y6, Y6
	VPADDD   Y11, Y7, Y7
	VPXOR    Y0, Y4, Y4
	VPXOR    Y5, Y1, Y1
	VPXOR    Y6, Y2, Y2
	VPXOR    Y7, Y3, Y3
	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
	ADDQ     16(DI), R10
	ADCQ     24(DI), R11
	ADCQ     $0x01, R12
	VPADDD   Y4, Y12, Y12
	VPADDD   Y1, Y13, Y13
	VPADDD   Y2, Y8, Y8
	VPADDD   Y3, Y15, Y15
	MOVQ     (BP), DX
	MOVQ     DX, R15
	MULXQ    R10, R13, R14
	IMULQ    R12, R15
	MULXQ    R11, AX, DX
	ADDQ     AX, R14
	ADCQ     DX, R15
	VPXOR    Y12, Y14, Y14
	VPXOR    Y13, Y9, Y9
	VPXOR    Y8, Y10, Y10
	VPXOR    Y15, Y11, Y11
	VMOVDQA  Y15, 224(BP)
	VPSLLD   $0x07, Y14, Y15
	VPSRLD   $0x19, Y14, Y14
	VPXOR    Y15, Y14, Y14
	VPSLLD   $0x07, Y9, Y15
	VPSRLD   $0x19, Y9, Y9
	VPXOR    Y15, Y9, Y9
	VPSLLD   $0x07, Y10, Y15
	VPSRLD   $0x19, Y10, Y10
	VPXOR    Y15, Y10, Y10
	VPSLLD   $0x07, Y11, Y15
	VPSRLD   $0x19, Y11, Y11
	VPXOR    Y15, Y11, Y11
	VMOVDQA  224(BP), Y15
	MOVQ     8(BP), DX
	MULXQ    R10, R10, AX
	ADDQ     R10, R14
	MULXQ    R11, R11, R8
	ADCQ     R11, R15
	ADCQ     $0x00, R8
	VPALIGNR $0x04, Y14, Y14, Y14
	VPALIGNR $0x04, Y9, Y9, Y9
	VPALIGNR $0x04, Y10, Y10, Y10
	VPALIGNR $0x04, Y11, Y11, Y11
	VPALIGNR $0x08, Y12, Y12, Y12
	VPALIGNR $0x08, Y13, Y13, Y13
	VPALIGNR $0x08, Y8, Y8, Y8
	VPALIGNR $0x08, Y15, Y15, Y15
	VPALIGNR $0x0c, Y4, Y4, Y4
	VPALIGNR $0x0c, Y1, Y1, Y1
	VPALIGNR $0x0c, Y2, Y2, Y2
	VPALIGNR $0x0c, Y3, Y3, Y3
	VPADDD   Y14, Y0, Y0
	VPADDD   Y9, Y5, Y5
	VPADDD   Y10, Y6, Y6
	VPADDD   Y11, Y7, Y7
	IMULQ    R12, DX
	ADDQ     AX, R15
	ADCQ     DX, R8
	VPXOR    Y0, Y4, Y4
	VPXOR    Y5, Y1, Y1
	VPXOR    Y6, Y2, Y2
	VPXOR    Y7, Y3, Y3
	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
	MOVQ     R13, R10
	MOVQ     R14, R11
	MOVQ     R15, R12
	ANDQ     $0x03, R12
	MOVQ     R15, R13
	ANDQ     $-4, R13
	MOVQ     R8, R14
	SHRQ     $0x02, R8, R15
	SHRQ     $0x02, R8
	ADDQ     R13, R10
	ADCQ     R14, R11
	ADCQ     $0x00, R12
	ADDQ     R15, R10
	ADCQ     R8, R11
	ADCQ     $0x00, R12
	VPADDD   Y4, Y12, Y12
	VPADDD   Y1, Y13, Y13
	VPADDD   Y2, Y8, Y8
	VPADDD   Y3, Y15, Y15
	VPXOR    Y12, Y14, Y14
	VPXOR    Y13, Y9, Y9
	VPXOR    Y8, Y10, Y10
	VPXOR    Y15, Y11, Y11
	ADDQ     32(DI), R10
	ADCQ     40(DI), R11
	ADCQ     $0x01, R12
	LEAQ     48(DI), DI
	VMOVDQA  Y15, 224(BP)
	VPSLLD   $0x0c, Y14, Y15
	VPSRLD   $0x14, Y14, Y14
	VPXOR    Y15, Y14, Y14
	VPSLLD   $0x0c, Y9, Y15
	VPSRLD   $0x14, Y9, Y9
	VPXOR    Y15, Y9, Y9
	VPSLLD   $0x0c, Y10, Y15
	VPSRLD   $0x14, Y10, Y10
	VPXOR    Y15, Y10, Y10
	VPSLLD   $0x0c, Y11, Y15
	VPSRLD   $0x14, Y11, Y11
	VPXOR    Y15, Y11, Y11
	VMOVDQA  224(BP), Y15
	MOVQ     (BP), DX
	MOVQ     DX, R15
	MULXQ    R10, R13, R14
	IMULQ    R12, R15
	MULXQ    R11, AX, DX
	ADDQ     AX, R14
	ADCQ     DX, R15
	VPADDD   Y14, Y0, Y0
	VPADDD   Y9, Y5, Y5
	VPADDD   Y10, Y6, Y6
	VPADDD   Y11, Y7, Y7
	VPXOR    Y0, Y4, Y4
	VPXOR    Y5, Y1, Y1
	VPXOR    Y6, Y2, Y2
	VPXOR    Y7, Y3, Y3
	MOVQ     8(BP), DX
	MULXQ    R10, R10, AX
	ADDQ     R10, R14
	MULXQ    R11, R11, R8
	ADCQ     R11, R15
	ADCQ     $0x00, R8
	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
	VPADDD   Y4, Y12, Y12
	VPADDD   Y1, Y13, Y13
	VPADDD   Y2, Y8, Y8
	VPADDD   Y3, Y15, Y15
	IMULQ    R12, DX
	ADDQ     AX, R15
	ADCQ     DX, R8
	VPXOR    Y12, Y14, Y14
	VPXOR    Y13, Y9, Y9
	VPXOR    Y8, Y10, Y10
	VPXOR    Y15, Y11, Y11
	VMOVDQA  Y15, 224(BP)
	VPSLLD   $0x07, Y14, Y15
	VPSRLD   $0x19, Y14, Y14
	VPXOR    Y15, Y14, Y14
	VPSLLD   $0x07, Y9, Y15
	VPSRLD   $0x19, Y9, Y9
	VPXOR    Y15, Y9, Y9
	VPSLLD   $0x07, Y10, Y15
	VPSRLD   $0x19, Y10, Y10
	VPXOR    Y15, Y10, Y10
	VPSLLD   $0x07, Y11, Y15
	VPSRLD   $0x19, Y11, Y11
	VPXOR    Y15, Y11, Y11
	VMOVDQA  224(BP), Y15
	MOVQ     R13, R10
	MOVQ     R14, R11
	MOVQ     R15, R12
	ANDQ     $0x03, R12
	MOVQ     R15, R13
	ANDQ     $-4, R13
	MOVQ     R8, R14
	SHRQ     $0x02, R8, R15
	SHRQ     $0x02, R8
	ADDQ     R13, R10
	ADCQ     R14, R11
	ADCQ     $0x00, R12
	ADDQ     R15, R10
	ADCQ     R8, R11
	ADCQ     $0x00, R12
	VPALIGNR $0x0c, Y14, Y14, Y14
	VPALIGNR $0x0c, Y9, Y9, Y9
	VPALIGNR $0x0c, Y10, Y10, Y10
	VPALIGNR $0x0c, Y11, Y11, Y11
	VPALIGNR $0x08, Y12, Y12, Y12
	VPALIGNR $0x08, Y13, Y13, Y13
	VPALIGNR $0x08, Y8, Y8, Y8
	VPALIGNR $0x08, Y15, Y15, Y15
	VPALIGNR $0x04, Y4, Y4, Y4
	VPALIGNR $0x04, Y1, Y1, Y1
	VPALIGNR $0x04, Y2, Y2, Y2
	VPALIGNR $0x04, Y3, Y3, Y3
	DECQ     CX
	JNE      sealAVX2InternalLoop
	VPADDD   ·chacha20Constants<>+0(SB), Y0, Y0
	VPADDD   ·chacha20Constants<>+0(SB), Y5, Y5
	VPADDD   ·chacha20Constants<>+0(SB), Y6, Y6
	VPADDD   ·chacha20Constants<>+0(SB), Y7, Y7
	VPADDD   32(BP), Y14, Y14
	VPADDD   32(BP), Y9, Y9
	VPADDD   32(BP), Y10, Y10
	VPADDD   32(BP), Y11, Y11
	VPADDD   64(BP), Y12, Y12
	VPADDD   64(BP), Y13, Y13
	VPADDD   64(BP), Y8, Y8
	VPADDD   64(BP), Y15, Y15
	VPADDD   96(BP), Y4, Y4
	VPADDD   128(BP), Y1, Y1
	VPADDD   160(BP), Y2, Y2
	VPADDD   192(BP), Y3, Y3
	VMOVDQA  Y15, 224(BP)

	// We only hashed 480 of the 512 bytes available - hash the remaining 32 here
	ADDQ       (DI), R10
	ADCQ       8(DI), R11
	ADCQ       $0x01, R12
	MOVQ       (BP), DX
	MOVQ       DX, R15
	MULXQ      R10, R13, R14
	IMULQ      R12, R15
	MULXQ      R11, AX, DX
	ADDQ       AX, R14
	ADCQ       DX, R15
	MOVQ       8(BP), DX
	MULXQ      R10, R10, AX
	ADDQ       R10, R14
	MULXQ      R11, R11, R8
	ADCQ       R11, R15
	ADCQ       $0x00, R8
	IMULQ      R12, DX
	ADDQ       AX, R15
	ADCQ       DX, R8
	MOVQ       R13, R10
	MOVQ       R14, R11
	MOVQ       R15, R12
	ANDQ       $0x03, R12
	MOVQ       R15, R13
	ANDQ       $-4, R13
	MOVQ       R8, R14
	SHRQ       $0x02, R8, R15
	SHRQ       $0x02, R8
	ADDQ       R13, R10
	ADCQ       R14, R11
	ADCQ       $0x00, R12
	ADDQ       R15, R10
	ADCQ       R8, R11
	ADCQ       $0x00, R12
	LEAQ       32(DI), DI
	VPERM2I128 $0x02, Y0, Y14, Y15
	VPERM2I128 $0x13, Y0, Y14, Y14
	VPERM2I128 $0x02, Y12, Y4, Y0
	VPERM2I128 $0x13, Y12, Y4, Y12
	VPXOR      (SI), Y15, Y15
	VPXOR      32(SI), Y0, Y0
	VPXOR      64(SI), Y14, Y14
	VPXOR      96(SI), Y12, Y12
	VMOVDQU    Y15, (DI)
	VMOVDQU    Y0, 32(DI)
	VMOVDQU    Y14, 64(DI)
	VMOVDQU    Y12, 96(DI)
	VPERM2I128 $0x02, Y5, Y9, Y0
	VPERM2I128 $0x02, Y13, Y1, Y14
	VPERM2I128 $0x13, Y5, Y9, Y12
	VPERM2I128 $0x13, Y13, Y1, Y4
	VPXOR      128(SI), Y0, Y0
	VPXOR      160(SI), Y14, Y14
	VPXOR      192(SI), Y12, Y12
	VPXOR      224(SI), Y4, Y4
	VMOVDQU    Y0, 128(DI)
	VMOVDQU    Y14, 160(DI)
	VMOVDQU    Y12, 192(DI)
	VMOVDQU    Y4, 224(DI)

	// and here
	ADDQ       -16(DI), R10
	ADCQ       -8(DI), R11
	ADCQ       $0x01, R12
	MOVQ       (BP), DX
	MOVQ       DX, R15
	MULXQ      R10, R13, R14
	IMULQ      R12, R15
	MULXQ      R11, AX, DX
	ADDQ       AX, R14
	ADCQ       DX, R15
	MOVQ       8(BP), DX
	MULXQ      R10, R10, AX
	ADDQ       R10, R14
	MULXQ      R11, R11, R8
	ADCQ       R11, R15
	ADCQ       $0x00, R8
	IMULQ      R12, DX
	ADDQ       AX, R15
	ADCQ       DX, R8
	MOVQ       R13, R10
	MOVQ       R14, R11
	MOVQ       R15, R12
	ANDQ       $0x03, R12
	MOVQ       R15, R13
	ANDQ       $-4, R13
	MOVQ       R8, R14
	SHRQ       $0x02, R8, R15
	SHRQ       $0x02, R8
	ADDQ       R13, R10
	ADCQ       R14, R11
	ADCQ       $0x00, R12
	ADDQ       R15, R10
	ADCQ       R8, R11
	ADCQ       $0x00, R12
	VPERM2I128 $0x02, Y6, Y10, Y0
	VPERM2I128 $0x02, Y8, Y2, Y14
	VPERM2I128 $0x13, Y6, Y10, Y12
	VPERM2I128 $0x13, Y8, Y2, Y4
	VPXOR      256(SI), Y0, Y0
	VPXOR      288(SI), Y14, Y14
	VPXOR      320(SI), Y12, Y12
	VPXOR      352(SI), Y4, Y4
	VMOVDQU    Y0, 256(DI)
	VMOVDQU    Y14, 288(DI)
	VMOVDQU    Y12, 320(DI)
	VMOVDQU    Y4, 352(DI)
	VPERM2I128 $0x02, Y7, Y11, Y0
	VPERM2I128 $0x02, 224(BP), Y3, Y14
	VPERM2I128 $0x13, Y7, Y11, Y12
	VPERM2I128 $0x13, 224(BP), Y3, Y4
	VPXOR      384(SI), Y0, Y0
	VPXOR      416(SI), Y14, Y14
	VPXOR      448(SI), Y12, Y12
	VPXOR      480(SI), Y4, Y4
	VMOVDQU    Y0, 384(DI)
	VMOVDQU    Y14, 416(DI)
	VMOVDQU    Y12, 448(DI)
	VMOVDQU    Y4, 480(DI)
	LEAQ       512(SI), SI
	SUBQ       $0x00000200, BX
	CMPQ       BX, $0x00000200
	JG         sealAVX2MainLoop

	// Tail can only hash 480 bytes
	ADDQ  (DI), R10
	ADCQ  8(DI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), DX
	MOVQ  DX, R15
	MULXQ R10, R13, R14
	IMULQ R12, R15
	MULXQ R11, AX, DX
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), DX
	MULXQ R10, R10, AX
	ADDQ  R10, R14
	MULXQ R11, R11, R8
	ADCQ  R11, R15
	ADCQ  $0x00, R8
	IMULQ R12, DX
	ADDQ  AX, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	ADDQ  16(DI), R10
	ADCQ  24(DI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), DX
	MOVQ  DX, R15
	MULXQ R10, R13, R14
	IMULQ R12, R15
	MULXQ R11, AX, DX
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), DX
	MULXQ R10, R10, AX
	ADDQ  R10, R14
	MULXQ R11, R11, R8
	ADCQ  R11, R15
	ADCQ  $0x00, R8
	IMULQ R12, DX
	ADDQ  AX, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  32(DI), DI
	MOVQ  $0x0000000a, CX
	MOVQ  $0x00000000, R9
	CMPQ  BX, $0x80
	JBE   sealAVX2Tail128
	CMPQ  BX, $0x00000100
	JBE   sealAVX2Tail256
	CMPQ  BX, $0x00000180
	JBE   sealAVX2Tail384
	JMP   sealAVX2Tail512

seal192AVX2:
	VMOVDQA Y0, Y5
	VMOVDQA Y14, Y9
	VMOVDQA Y12, Y13
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
	VMOVDQA Y0, Y6
	VMOVDQA Y14, Y10
	VMOVDQA Y12, Y8
	VMOVDQA Y4, Y2
	VMOVDQA Y1, Y15
	MOVQ    $0x0000000a, R9

sealAVX2192InnerCipherLoop:
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x0c, Y14, Y3
	VPSRLD     $0x14, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x07, Y14, Y3
	VPSRLD     $0x19, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x0c, Y9, Y3
	VPSRLD     $0x14, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x07, Y9, Y3
	VPSRLD     $0x19, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPALIGNR   $0x04, Y14, Y14, Y14
	VPALIGNR   $0x04, Y9, Y9, Y9
	VPALIGNR   $0x08, Y12, Y12, Y12
	VPALIGNR   $0x08, Y13, Y13, Y13
	VPALIGNR   $0x0c, Y4, Y4, Y4
	VPALIGNR   $0x0c, Y1, Y1, Y1
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x0c, Y14, Y3
	VPSRLD     $0x14, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x07, Y14, Y3
	VPSRLD     $0x19, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x0c, Y9, Y3
	VPSRLD     $0x14, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x07, Y9, Y3
	VPSRLD     $0x19, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPALIGNR   $0x0c, Y14, Y14, Y14
	VPALIGNR   $0x0c, Y9, Y9, Y9
	VPALIGNR   $0x08, Y12, Y12, Y12
	VPALIGNR   $0x08, Y13, Y13, Y13
	VPALIGNR   $0x04, Y4, Y4, Y4
	VPALIGNR   $0x04, Y1, Y1, Y1
	DECQ       R9
	JNE        sealAVX2192InnerCipherLoop
	VPADDD     Y6, Y0, Y0
	VPADDD     Y6, Y5, Y5
	VPADDD     Y10, Y14, Y14
	VPADDD     Y10, Y9, Y9
	VPADDD     Y8, Y12, Y12
	VPADDD     Y8, Y13, Y13
	VPADDD     Y2, Y4, Y4
	VPADDD     Y15, Y1, Y1
	VPERM2I128 $0x02, Y0, Y14, Y3

	// Clamp and store poly key
	VPAND   ·polyClampMask<>+0(SB), Y3, Y3
	VMOVDQA Y3, (BP)

	// Stream for up to 192 bytes
	VPERM2I128 $0x13, Y0, Y14, Y0
	VPERM2I128 $0x13, Y12, Y4, Y14
	VPERM2I128 $0x02, Y5, Y9, Y12
	VPERM2I128 $0x02, Y13, Y1, Y4
	VPERM2I128 $0x13, Y5, Y9, Y5
	VPERM2I128 $0x13, Y13, Y1, Y9

sealAVX2ShortSeal:
	// Hash aad
	MOVQ ad_len+80(FP), R9
	CALL polyHashADInternal<>(SB)
	XORQ CX, CX

sealAVX2SealHash:
	// itr1 holds the number of bytes encrypted but not yet hashed
	CMPQ  CX, $0x10
	JB    sealAVX2ShortSealLoop
	ADDQ  (DI), R10
	ADCQ  8(DI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	SUBQ  $0x10, CX
	ADDQ  $0x10, DI
	JMP   sealAVX2SealHash

sealAVX2ShortSealLoop:
	CMPQ BX, $0x20
	JB   sealAVX2ShortTail32
	SUBQ $0x20, BX

	// Load for encryption
	VPXOR   (SI), Y0, Y0
	VMOVDQU Y0, (DI)
	LEAQ    32(SI), SI

	// Now can hash
	ADDQ  (DI), R10
	ADCQ  8(DI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), DX
	MOVQ  DX, R15
	MULXQ R10, R13, R14
	IMULQ R12, R15
	MULXQ R11, AX, DX
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), DX
	MULXQ R10, R10, AX
	ADDQ  R10, R14
	MULXQ R11, R11, R8
	ADCQ  R11, R15
	ADCQ  $0x00, R8
	IMULQ R12, DX
	ADDQ  AX, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	ADDQ  16(DI), R10
	ADCQ  24(DI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), DX
	MOVQ  DX, R15
	MULXQ R10, R13, R14
	IMULQ R12, R15
	MULXQ R11, AX, DX
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), DX
	MULXQ R10, R10, AX
	ADDQ  R10, R14
	MULXQ R11, R11, R8
	ADCQ  R11, R15
	ADCQ  $0x00, R8
	IMULQ R12, DX
	ADDQ  AX, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  32(DI), DI

	// Shift stream left
	VMOVDQA Y14, Y0
	VMOVDQA Y12, Y14
	VMOVDQA Y4, Y12
	VMOVDQA Y5, Y4
	VMOVDQA Y9, Y5
	VMOVDQA Y13, Y9
	VMOVDQA Y1, Y13
	VMOVDQA Y6, Y1
	VMOVDQA Y10, Y6
	JMP     sealAVX2ShortSealLoop

sealAVX2ShortTail32:
	CMPQ    BX, $0x10
	VMOVDQA X0, X1
	JB      sealAVX2ShortDone
	SUBQ    $0x10, BX

	// Load for encryption
	VPXOR   (SI), X0, X12
	VMOVDQU X12, (DI)
	LEAQ    16(SI), SI

	// Hash
	ADDQ       (DI), R10
	ADCQ       8(DI), R11
	ADCQ       $0x01, R12
	MOVQ       (BP), DX
	MOVQ       DX, R15
	MULXQ      R10, R13, R14
	IMULQ      R12, R15
	MULXQ      R11, AX, DX
	ADDQ       AX, R14
	ADCQ       DX, R15
	MOVQ       8(BP), DX
	MULXQ      R10, R10, AX
	ADDQ       R10, R14
	MULXQ      R11, R11, R8
	ADCQ       R11, R15
	ADCQ       $0x00, R8
	IMULQ      R12, DX
	ADDQ       AX, R15
	ADCQ       DX, R8
	MOVQ       R13, R10
	MOVQ       R14, R11
	MOVQ       R15, R12
	ANDQ       $0x03, R12
	MOVQ       R15, R13
	ANDQ       $-4, R13
	MOVQ       R8, R14
	SHRQ       $0x02, R8, R15
	SHRQ       $0x02, R8
	ADDQ       R13, R10
	ADCQ       R14, R11
	ADCQ       $0x00, R12
	ADDQ       R15, R10
	ADCQ       R8, R11
	ADCQ       $0x00, R12
	LEAQ       16(DI), DI
	VPERM2I128 $0x11, Y0, Y0, Y0
	VMOVDQA    X0, X1

sealAVX2ShortDone:
	VZEROUPPER
	JMP sealSSETail

seal320AVX2:
	VMOVDQA Y0, Y5
	VMOVDQA Y14, Y9
	VMOVDQA Y12, Y13
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
	VMOVDQA Y0, Y6
	VMOVDQA Y14, Y10
	VMOVDQA Y12, Y8
	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
	VMOVDQA Y14, Y7
	VMOVDQA Y12, Y11
	VMOVDQA Y4, Y15
	MOVQ    $0x0000000a, R9

sealAVX2320InnerCipherLoop:
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x0c, Y14, Y3
	VPSRLD   $0x14, Y14, Y14
	VPXOR    Y3, Y14, Y14
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x07, Y14, Y3
	VPSRLD   $0x19, Y14, Y14
	VPXOR    Y3, Y14, Y14
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x0c, Y9, Y3
	VPSRLD   $0x14, Y9, Y9
	VPXOR    Y3, Y9, Y9
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x07, Y9, Y3
	VPSRLD   $0x19, Y9, Y9
	VPXOR    Y3, Y9, Y9
	VPADDD   Y10, Y6, Y6
	VPXOR    Y6, Y2, Y2
	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
	VPADDD   Y2, Y8, Y8
	VPXOR    Y8, Y10, Y10
	VPSLLD   $0x0c, Y10, Y3
	VPSRLD   $0x14, Y10, Y10
	VPXOR    Y3, Y10, Y10
	VPADDD   Y10, Y6, Y6
	VPXOR    Y6, Y2, Y2
	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
	VPADDD   Y2, Y8, Y8
	VPXOR    Y8, Y10, Y10
	VPSLLD   $0x07, Y10, Y3
	VPSRLD   $0x19, Y10, Y10
	VPXOR    Y3, Y10, Y10
	VPALIGNR $0x04, Y14, Y14, Y14
	VPALIGNR $0x04, Y9, Y9, Y9
	VPALIGNR $0x04, Y10, Y10, Y10
	VPALIGNR $0x08, Y12, Y12, Y12
	VPALIGNR $0x08, Y13, Y13, Y13
	VPALIGNR $0x08, Y8, Y8, Y8
	VPALIGNR $0x0c, Y4, Y4, Y4
	VPALIGNR $0x0c, Y1, Y1, Y1
	VPALIGNR $0x0c, Y2, Y2, Y2
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x0c, Y14, Y3
	VPSRLD   $0x14, Y14, Y14
	VPXOR    Y3, Y14, Y14
	VPADDD   Y14, Y0, Y0
	VPXOR    Y0, Y4, Y4
	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
	VPADDD   Y4, Y12, Y12
	VPXOR    Y12, Y14, Y14
	VPSLLD   $0x07, Y14, Y3
	VPSRLD   $0x19, Y14, Y14
	VPXOR    Y3, Y14, Y14
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x0c, Y9, Y3
	VPSRLD   $0x14, Y9, Y9
	VPXOR    Y3, Y9, Y9
	VPADDD   Y9, Y5, Y5
	VPXOR    Y5, Y1, Y1
	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
	VPADDD   Y1, Y13, Y13
	VPXOR    Y13, Y9, Y9
	VPSLLD   $0x07, Y9, Y3
	VPSRLD   $0x19, Y9, Y9
	VPXOR    Y3, Y9, Y9
	VPADDD   Y10, Y6, Y6
	VPXOR    Y6, Y2, Y2
	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
	VPADDD   Y2, Y8, Y8
	VPXOR    Y8, Y10, Y10
	VPSLLD   $0x0c, Y10, Y3
	VPSRLD   $0x14, Y10, Y10
	VPXOR    Y3, Y10, Y10
	VPADDD   Y10, Y6, Y6
	VPXOR    Y6, Y2, Y2
	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
	VPADDD   Y2, Y8, Y8
	VPXOR    Y8, Y10, Y10
	VPSLLD   $0x07, Y10, Y3
	VPSRLD   $0x19, Y10, Y10
	VPXOR    Y3, Y10, Y10
	VPALIGNR $0x0c, Y14, Y14, Y14
	VPALIGNR $0x0c, Y9, Y9, Y9
	VPALIGNR $0x0c, Y10, Y10, Y10
	VPALIGNR $0x08, Y12, Y12, Y12
	VPALIGNR $0x08, Y13, Y13, Y13
	VPALIGNR $0x08, Y8, Y8, Y8
	VPALIGNR $0x04, Y4, Y4, Y4
	VPALIGNR $0x04, Y1, Y1, Y1
	VPALIGNR $0x04, Y2, Y2, Y2
	DECQ     R9
	JNE      sealAVX2320InnerCipherLoop
	VMOVDQA  ·chacha20Constants<>+0(SB), Y3
	VPADDD   Y3, Y0, Y0
	VPADDD   Y3, Y5, Y5
	VPADDD   Y3, Y6, Y6
	VPADDD   Y7, Y14, Y14
	VPADDD   Y7, Y9, Y9
	VPADDD   Y7, Y10, Y10
	VPADDD   Y11, Y12, Y12
	VPADDD   Y11, Y13, Y13
	VPADDD   Y11, Y8, Y8
	VMOVDQA  ·avx2IncMask<>+0(SB), Y3
	VPADDD   Y15, Y4, Y4
	VPADDD   Y3, Y15, Y15
	VPADDD   Y15, Y1, Y1
	VPADDD   Y3, Y15, Y15
	VPADDD   Y15, Y2, Y2

	// Clamp and store poly key
	VPERM2I128 $0x02, Y0, Y14, Y3
	VPAND      ·polyClampMask<>+0(SB), Y3, Y3
	VMOVDQA    Y3, (BP)

	// Stream for up to 320 bytes
	VPERM2I128 $0x13, Y0, Y14, Y0
	VPERM2I128 $0x13, Y12, Y4, Y14
	VPERM2I128 $0x02, Y5, Y9, Y12
	VPERM2I128 $0x02, Y13, Y1, Y4
	VPERM2I128 $0x13, Y5, Y9, Y5
	VPERM2I128 $0x13, Y13, Y1, Y9
	VPERM2I128 $0x02, Y6, Y10, Y13
	VPERM2I128 $0x02, Y8, Y2, Y1
	VPERM2I128 $0x13, Y6, Y10, Y6
	VPERM2I128 $0x13, Y8, Y2, Y10
	JMP        sealAVX2ShortSeal

sealAVX2Tail128:
	VMOVDQA ·chacha20Constants<>+0(SB), Y0
	VMOVDQA 32(BP), Y14
	VMOVDQA 64(BP), Y12
	VMOVDQA 192(BP), Y4
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
	VMOVDQA Y4, Y1

sealAVX2Tail128LoopA:
	ADDQ  (DI), R10
	ADCQ  8(DI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(DI), DI

sealAVX2Tail128LoopB:
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x0c, Y14, Y3
	VPSRLD     $0x14, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x07, Y14, Y3
	VPSRLD     $0x19, Y14, Y14
	VPXOR      Y3, Y14, Y14
	ADDQ       (DI), R10
	ADCQ       8(DI), R11
	ADCQ       $0x01, R12
	MOVQ       (BP), AX
	MOVQ       AX, R15
	MULQ       R10
	MOVQ       AX, R13
	MOVQ       DX, R14
	MOVQ       (BP), AX
	MULQ       R11
	IMULQ      R12, R15
	ADDQ       AX, R14
	ADCQ       DX, R15
	MOVQ       8(BP), AX
	MOVQ       AX, R8
	MULQ       R10
	ADDQ       AX, R14
	ADCQ       $0x00, DX
	MOVQ       DX, R10
	MOVQ       8(BP), AX
	MULQ       R11
	ADDQ       AX, R15
	ADCQ       $0x00, DX
	IMULQ      R12, R8
	ADDQ       R10, R15
	ADCQ       DX, R8
	MOVQ       R13, R10
	MOVQ       R14, R11
	MOVQ       R15, R12
	ANDQ       $0x03, R12
	MOVQ       R15, R13
	ANDQ       $-4, R13
	MOVQ       R8, R14
	SHRQ       $0x02, R8, R15
	SHRQ       $0x02, R8
	ADDQ       R13, R10
	ADCQ       R14, R11
	ADCQ       $0x00, R12
	ADDQ       R15, R10
	ADCQ       R8, R11
	ADCQ       $0x00, R12
	VPALIGNR   $0x04, Y14, Y14, Y14
	VPALIGNR   $0x08, Y12, Y12, Y12
	VPALIGNR   $0x0c, Y4, Y4, Y4
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x0c, Y14, Y3
	VPSRLD     $0x14, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x07, Y14, Y3
	VPSRLD     $0x19, Y14, Y14
	VPXOR      Y3, Y14, Y14
	ADDQ       16(DI), R10
	ADCQ       24(DI), R11
	ADCQ       $0x01, R12
	MOVQ       (BP), AX
	MOVQ       AX, R15
	MULQ       R10
	MOVQ       AX, R13
	MOVQ       DX, R14
	MOVQ       (BP), AX
	MULQ       R11
	IMULQ      R12, R15
	ADDQ       AX, R14
	ADCQ       DX, R15
	MOVQ       8(BP), AX
	MOVQ       AX, R8
	MULQ       R10
	ADDQ       AX, R14
	ADCQ       $0x00, DX
	MOVQ       DX, R10
	MOVQ       8(BP), AX
	MULQ       R11
	ADDQ       AX, R15
	ADCQ       $0x00, DX
	IMULQ      R12, R8
	ADDQ       R10, R15
	ADCQ       DX, R8
	MOVQ       R13, R10
	MOVQ       R14, R11
	MOVQ       R15, R12
	ANDQ       $0x03, R12
	MOVQ       R15, R13
	ANDQ       $-4, R13
	MOVQ       R8, R14
	SHRQ       $0x02, R8, R15
	SHRQ       $0x02, R8
	ADDQ       R13, R10
	ADCQ       R14, R11
	ADCQ       $0x00, R12
	ADDQ       R15, R10
	ADCQ       R8, R11
	ADCQ       $0x00, R12
	LEAQ       32(DI), DI
	VPALIGNR   $0x0c, Y14, Y14, Y14
	VPALIGNR   $0x08, Y12, Y12, Y12
	VPALIGNR   $0x04, Y4, Y4, Y4
	DECQ       CX
	JG         sealAVX2Tail128LoopA
	DECQ       R9
	JGE        sealAVX2Tail128LoopB
	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y5
	VPADDD     32(BP), Y14, Y9
	VPADDD     64(BP), Y12, Y13
	VPADDD     Y1, Y4, Y1
	VPERM2I128 $0x02, Y5, Y9, Y0
	VPERM2I128 $0x02, Y13, Y1, Y14
	VPERM2I128 $0x13, Y5, Y9, Y12
	VPERM2I128 $0x13, Y13, Y1, Y4
	JMP        sealAVX2ShortSealLoop

sealAVX2Tail256:
	VMOVDQA ·chacha20Constants<>+0(SB), Y0
	VMOVDQA ·chacha20Constants<>+0(SB), Y5
	VMOVDQA 32(BP), Y14
	VMOVDQA 32(BP), Y9
	VMOVDQA 64(BP), Y12
	VMOVDQA 64(BP), Y13
	VMOVDQA 192(BP), Y4
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
	VMOVDQA Y4, Y7
	VMOVDQA Y1, Y11

sealAVX2Tail256LoopA:
	ADDQ  (DI), R10
	ADCQ  8(DI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(DI), DI

sealAVX2Tail256LoopB:
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x0c, Y14, Y3
	VPSRLD     $0x14, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x07, Y14, Y3
	VPSRLD     $0x19, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x0c, Y9, Y3
	VPSRLD     $0x14, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x07, Y9, Y3
	VPSRLD     $0x19, Y9, Y9
	VPXOR      Y3, Y9, Y9
	ADDQ       (DI), R10
	ADCQ       8(DI), R11
	ADCQ       $0x01, R12
	MOVQ       (BP), AX
	MOVQ       AX, R15
	MULQ       R10
	MOVQ       AX, R13
	MOVQ       DX, R14
	MOVQ       (BP), AX
	MULQ       R11
	IMULQ      R12, R15
	ADDQ       AX, R14
	ADCQ       DX, R15
	MOVQ       8(BP), AX
	MOVQ       AX, R8
	MULQ       R10
	ADDQ       AX, R14
	ADCQ       $0x00, DX
	MOVQ       DX, R10
	MOVQ       8(BP), AX
	MULQ       R11
	ADDQ       AX, R15
	ADCQ       $0x00, DX
	IMULQ      R12, R8
	ADDQ       R10, R15
	ADCQ       DX, R8
	MOVQ       R13, R10
	MOVQ       R14, R11
	MOVQ       R15, R12
	ANDQ       $0x03, R12
	MOVQ       R15, R13
	ANDQ       $-4, R13
	MOVQ       R8, R14
	SHRQ       $0x02, R8, R15
	SHRQ       $0x02, R8
	ADDQ       R13, R10
	ADCQ       R14, R11
	ADCQ       $0x00, R12
	ADDQ       R15, R10
	ADCQ       R8, R11
	ADCQ       $0x00, R12
	VPALIGNR   $0x04, Y14, Y14, Y14
	VPALIGNR   $0x04, Y9, Y9, Y9
	VPALIGNR   $0x08, Y12, Y12, Y12
	VPALIGNR   $0x08, Y13, Y13, Y13
	VPALIGNR   $0x0c, Y4, Y4, Y4
	VPALIGNR   $0x0c, Y1, Y1, Y1
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x0c, Y14, Y3
	VPSRLD     $0x14, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x07, Y14, Y3
	VPSRLD     $0x19, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x0c, Y9, Y3
	VPSRLD     $0x14, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x07, Y9, Y3
	VPSRLD     $0x19, Y9, Y9
	VPXOR      Y3, Y9, Y9
	ADDQ       16(DI), R10
	ADCQ       24(DI), R11
	ADCQ       $0x01, R12
	MOVQ       (BP), AX
	MOVQ       AX, R15
	MULQ       R10
	MOVQ       AX, R13
	MOVQ       DX, R14
	MOVQ       (BP), AX
	MULQ       R11
	IMULQ      R12, R15
	ADDQ       AX, R14
	ADCQ       DX, R15
	MOVQ       8(BP), AX
	MOVQ       AX, R8
	MULQ       R10
	ADDQ       AX, R14
	ADCQ       $0x00, DX
	MOVQ       DX, R10
	MOVQ       8(BP), AX
	MULQ       R11
	ADDQ       AX, R15
	ADCQ       $0x00, DX
	IMULQ      R12, R8
	ADDQ       R10, R15
	ADCQ       DX, R8
	MOVQ       R13, R10
	MOVQ       R14, R11
	MOVQ       R15, R12
	ANDQ       $0x03, R12
	MOVQ       R15, R13
	ANDQ       $-4, R13
	MOVQ       R8, R14
	SHRQ       $0x02, R8, R15
	SHRQ       $0x02, R8
	ADDQ       R13, R10
	ADCQ       R14, R11
	ADCQ       $0x00, R12
	ADDQ       R15, R10
	ADCQ       R8, R11
	ADCQ       $0x00, R12
	LEAQ       32(DI), DI
	VPALIGNR   $0x0c, Y14, Y14, Y14
	VPALIGNR   $0x0c, Y9, Y9, Y9
	VPALIGNR   $0x08, Y12, Y12, Y12
	VPALIGNR   $0x08, Y13, Y13, Y13
	VPALIGNR   $0x04, Y4, Y4, Y4
	VPALIGNR   $0x04, Y1, Y1, Y1
	DECQ       CX
	JG         sealAVX2Tail256LoopA
	DECQ       R9
	JGE        sealAVX2Tail256LoopB
	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
	VPADDD     32(BP), Y14, Y14
	VPADDD     32(BP), Y9, Y9
	VPADDD     64(BP), Y12, Y12
	VPADDD     64(BP), Y13, Y13
	VPADDD     Y7, Y4, Y4
	VPADDD     Y11, Y1, Y1
	VPERM2I128 $0x02, Y0, Y14, Y3
	VPERM2I128 $0x02, Y12, Y4, Y7
	VPERM2I128 $0x13, Y0, Y14, Y11
	VPERM2I128 $0x13, Y12, Y4, Y15
	VPXOR      (SI), Y3, Y3
	VPXOR      32(SI), Y7, Y7
	VPXOR      64(SI), Y11, Y11
	VPXOR      96(SI), Y15, Y15
	VMOVDQU    Y3, (DI)
	VMOVDQU    Y7, 32(DI)
	VMOVDQU    Y11, 64(DI)
	VMOVDQU    Y15, 96(DI)
	MOVQ       $0x00000080, CX
	LEAQ       128(SI), SI
	SUBQ       $0x80, BX
	VPERM2I128 $0x02, Y5, Y9, Y0
	VPERM2I128 $0x02, Y13, Y1, Y14
	VPERM2I128 $0x13, Y5, Y9, Y12
	VPERM2I128 $0x13, Y13, Y1, Y4
	JMP        sealAVX2SealHash

sealAVX2Tail384:
	VMOVDQA ·chacha20Constants<>+0(SB), Y0
	VMOVDQA Y0, Y5
	VMOVDQA Y0, Y6
	VMOVDQA 32(BP), Y14
	VMOVDQA Y14, Y9
	VMOVDQA Y14, Y10
	VMOVDQA 64(BP), Y12
	VMOVDQA Y12, Y13
	VMOVDQA Y12, Y8
	VMOVDQA 192(BP), Y4
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
	VMOVDQA Y4, Y7
	VMOVDQA Y1, Y11
	VMOVDQA Y2, Y15

sealAVX2Tail384LoopA:
	ADDQ  (DI), R10
	ADCQ  8(DI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(DI), DI

sealAVX2Tail384LoopB:
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x0c, Y14, Y3
	VPSRLD     $0x14, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x07, Y14, Y3
	VPSRLD     $0x19, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x0c, Y9, Y3
	VPSRLD     $0x14, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x07, Y9, Y3
	VPSRLD     $0x19, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPADDD     Y10, Y6, Y6
	VPXOR      Y6, Y2, Y2
	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
	VPADDD     Y2, Y8, Y8
	VPXOR      Y8, Y10, Y10
	VPSLLD     $0x0c, Y10, Y3
	VPSRLD     $0x14, Y10, Y10
	VPXOR      Y3, Y10, Y10
	VPADDD     Y10, Y6, Y6
	VPXOR      Y6, Y2, Y2
	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
	VPADDD     Y2, Y8, Y8
	VPXOR      Y8, Y10, Y10
	VPSLLD     $0x07, Y10, Y3
	VPSRLD     $0x19, Y10, Y10
	VPXOR      Y3, Y10, Y10
	ADDQ       (DI), R10
	ADCQ       8(DI), R11
	ADCQ       $0x01, R12
	MOVQ       (BP), AX
	MOVQ       AX, R15
	MULQ       R10
	MOVQ       AX, R13
	MOVQ       DX, R14
	MOVQ       (BP), AX
	MULQ       R11
	IMULQ      R12, R15
	ADDQ       AX, R14
	ADCQ       DX, R15
	MOVQ       8(BP), AX
	MOVQ       AX, R8
	MULQ       R10
	ADDQ       AX, R14
	ADCQ       $0x00, DX
	MOVQ       DX, R10
	MOVQ       8(BP), AX
	MULQ       R11
	ADDQ       AX, R15
	ADCQ       $0x00, DX
	IMULQ      R12, R8
	ADDQ       R10, R15
	ADCQ       DX, R8
	MOVQ       R13, R10
	MOVQ       R14, R11
	MOVQ       R15, R12
	ANDQ       $0x03, R12
	MOVQ       R15, R13
	ANDQ       $-4, R13
	MOVQ       R8, R14
	SHRQ       $0x02, R8, R15
	SHRQ       $0x02, R8
	ADDQ       R13, R10
	ADCQ       R14, R11
	ADCQ       $0x00, R12
	ADDQ       R15, R10
	ADCQ       R8, R11
	ADCQ       $0x00, R12
	VPALIGNR   $0x04, Y14, Y14, Y14
	VPALIGNR   $0x04, Y9, Y9, Y9
	VPALIGNR   $0x04, Y10, Y10, Y10
	VPALIGNR   $0x08, Y12, Y12, Y12
	VPALIGNR   $0x08, Y13, Y13, Y13
	VPALIGNR   $0x08, Y8, Y8, Y8
	VPALIGNR   $0x0c, Y4, Y4, Y4
	VPALIGNR   $0x0c, Y1, Y1, Y1
	VPALIGNR   $0x0c, Y2, Y2, Y2
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x0c, Y14, Y3
	VPSRLD     $0x14, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y14, Y0, Y0
	VPXOR      Y0, Y4, Y4
	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
	VPADDD     Y4, Y12, Y12
	VPXOR      Y12, Y14, Y14
	VPSLLD     $0x07, Y14, Y3
	VPSRLD     $0x19, Y14, Y14
	VPXOR      Y3, Y14, Y14
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x0c, Y9, Y3
	VPSRLD     $0x14, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPADDD     Y9, Y5, Y5
	VPXOR      Y5, Y1, Y1
	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
	VPADDD     Y1, Y13, Y13
	VPXOR      Y13, Y9, Y9
	VPSLLD     $0x07, Y9, Y3
	VPSRLD     $0x19, Y9, Y9
	VPXOR      Y3, Y9, Y9
	VPADDD     Y10, Y6, Y6
	VPXOR      Y6, Y2, Y2
	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
	VPADDD     Y2, Y8, Y8
	VPXOR      Y8, Y10, Y10
	VPSLLD     $0x0c, Y10, Y3
	VPSRLD     $0x14, Y10, Y10
	VPXOR      Y3, Y10, Y10
	VPADDD     Y10, Y6, Y6
	VPXOR      Y6, Y2, Y2
	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
	VPADDD     Y2, Y8, Y8
	VPXOR      Y8, Y10, Y10
	VPSLLD     $0x07, Y10, Y3
	VPSRLD     $0x19, Y10, Y10
	VPXOR      Y3, Y10, Y10
	ADDQ       16(DI), R10
	ADCQ       24(DI), R11
	ADCQ       $0x01, R12
	MOVQ       (BP), AX
	MOVQ       AX, R15
	MULQ       R10
	MOVQ       AX, R13
	MOVQ       DX, R14
	MOVQ       (BP), AX
	MULQ       R11
	IMULQ      R12, R15
	ADDQ       AX, R14
	ADCQ       DX, R15
	MOVQ       8(BP), AX
	MOVQ       AX, R8
	MULQ       R10
	ADDQ       AX, R14
	ADCQ       $0x00, DX
	MOVQ       DX, R10
	MOVQ       8(BP), AX
	MULQ       R11
	ADDQ       AX, R15
	ADCQ       $0x00, DX
	IMULQ      R12, R8
	ADDQ       R10, R15
	ADCQ       DX, R8
	MOVQ       R13, R10
	MOVQ       R14, R11
	MOVQ       R15, R12
	ANDQ       $0x03, R12
	MOVQ       R15, R13
	ANDQ       $-4, R13
	MOVQ       R8, R14
	SHRQ       $0x02, R8, R15
	SHRQ       $0x02, R8
	ADDQ       R13, R10
	ADCQ       R14, R11
	ADCQ       $0x00, R12
	ADDQ       R15, R10
	ADCQ       R8, R11
	ADCQ       $0x00, R12
	LEAQ       32(DI), DI
	VPALIGNR   $0x0c, Y14, Y14, Y14
	VPALIGNR   $0x0c, Y9, Y9, Y9
	VPALIGNR   $0x0c, Y10, Y10, Y10
	VPALIGNR   $0x08, Y12, Y12, Y12
	VPALIGNR   $0x08, Y13, Y13, Y13
	VPALIGNR   $0x08, Y8, Y8, Y8
	VPALIGNR   $0x04, Y4, Y4, Y4
	VPALIGNR   $0x04, Y1, Y1, Y1
	VPALIGNR   $0x04, Y2, Y2, Y2
	DECQ       CX
	JG         sealAVX2Tail384LoopA
	DECQ       R9
	JGE        sealAVX2Tail384LoopB
	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
	VPADDD     32(BP), Y14, Y14
	VPADDD     32(BP), Y9, Y9
	VPADDD     32(BP), Y10, Y10
	VPADDD     64(BP), Y12, Y12
	VPADDD     64(BP), Y13, Y13
	VPADDD     64(BP), Y8, Y8
	VPADDD     Y7, Y4, Y4
	VPADDD     Y11, Y1, Y1
	VPADDD     Y15, Y2, Y2
	VPERM2I128 $0x02, Y0, Y14, Y3
	VPERM2I128 $0x02, Y12, Y4, Y7
	VPERM2I128 $0x13, Y0, Y14, Y11
	VPERM2I128 $0x13, Y12, Y4, Y15
	VPXOR      (SI), Y3, Y3
	VPXOR      32(SI), Y7, Y7
	VPXOR      64(SI), Y11, Y11
	VPXOR      96(SI), Y15, Y15
	VMOVDQU    Y3, (DI)
	VMOVDQU    Y7, 32(DI)
	VMOVDQU    Y11, 64(DI)
	VMOVDQU    Y15, 96(DI)
	VPERM2I128 $0x02, Y5, Y9, Y3
	VPERM2I128 $0x02, Y13, Y1, Y7
	VPERM2I128 $0x13, Y5, Y9, Y11
	VPERM2I128 $0x13, Y13, Y1, Y15
	VPXOR      128(SI), Y3, Y3
	VPXOR      160(SI), Y7, Y7
	VPXOR      192(SI), Y11, Y11
	VPXOR      224(SI), Y15, Y15
	VMOVDQU    Y3, 128(DI)
	VMOVDQU    Y7, 160(DI)
	VMOVDQU    Y11, 192(DI)
	VMOVDQU    Y15, 224(DI)
	MOVQ       $0x00000100, CX
	LEAQ       256(SI), SI
	SUBQ       $0x00000100, BX
	VPERM2I128 $0x02, Y6, Y10, Y0
	VPERM2I128 $0x02, Y8, Y2, Y14
	VPERM2I128 $0x13, Y6, Y10, Y12
	VPERM2I128 $0x13, Y8, Y2, Y4
	JMP        sealAVX2SealHash

sealAVX2Tail512:
	VMOVDQA ·chacha20Constants<>+0(SB), Y0
	VMOVDQA Y0, Y5
	VMOVDQA Y0, Y6
	VMOVDQA Y0, Y7
	VMOVDQA 32(BP), Y14
	VMOVDQA Y14, Y9
	VMOVDQA Y14, Y10
	VMOVDQA Y14, Y11
	VMOVDQA 64(BP), Y12
	VMOVDQA Y12, Y13
	VMOVDQA Y12, Y8
	VMOVDQA Y12, Y15
	VMOVDQA 192(BP), Y4
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
	VMOVDQA Y4, 96(BP)
	VMOVDQA Y1, 128(BP)
	VMOVDQA Y2, 160(BP)
	VMOVDQA Y3, 192(BP)

sealAVX2Tail512LoopA:
	ADDQ  (DI), R10
	ADCQ  8(DI), R11
	ADCQ  $0x01, R12
	MOVQ  (BP), AX
	MOVQ  AX, R15
	MULQ  R10
	MOVQ  AX, R13
	MOVQ  DX, R14
	MOVQ  (BP), AX
	MULQ  R11
	IMULQ R12, R15
	ADDQ  AX, R14
	ADCQ  DX, R15
	MOVQ  8(BP), AX
	MOVQ  AX, R8
	MULQ  R10
	ADDQ  AX, R14
	ADCQ  $0x00, DX
	MOVQ  DX, R10
	MOVQ  8(BP), AX
	MULQ  R11
	ADDQ  AX, R15
	ADCQ  $0x00, DX
	IMULQ R12, R8
	ADDQ  R10, R15
	ADCQ  DX, R8
	MOVQ  R13, R10
	MOVQ  R14, R11
	MOVQ  R15, R12
	ANDQ  $0x03, R12
	MOVQ  R15, R13
	ANDQ  $-4, R13
	MOVQ  R8, R14
	SHRQ  $0x02, R8, R15
	SHRQ  $0x02, R8
	ADDQ  R13, R10
	ADCQ  R14, R11
	ADCQ  $0x00, R12
	ADDQ  R15, R10
	ADCQ  R8, R11
	ADCQ  $0x00, R12
	LEAQ  16(DI), DI

sealAVX2Tail512LoopB:
	VPADDD     Y14, Y0, Y0
	VPADDD     Y9, Y5, Y5
	VPADDD     Y10, Y6, Y6
	VPADDD     Y11, Y7, Y7
	VPXOR      Y0, Y4, Y4
	VPXOR      Y5, Y1, Y1
	VPXOR      Y6, Y2, Y2
	VPXOR      Y7, Y3, Y3
	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
	VPSHUFB    ·rol16<>+0(SB), Y3, Y3
	VPADDD     Y4, Y12, Y12
	VPADDD     Y1, Y13, Y13
	VPADDD     Y2, Y8, Y8
	VPADDD     Y3, Y15, Y15
	VPXOR      Y12, Y14, Y14
	VPXOR      Y13, Y9, Y9
	VPXOR      Y8, Y10, Y10
	VPXOR      Y15, Y11, Y11
	VMOVDQA    Y15, 224(BP)
	VPSLLD     $0x0c, Y14, Y15
	VPSRLD     $0x14, Y14, Y14
	VPXOR      Y15, Y14, Y14
	VPSLLD     $0x0c, Y9, Y15
	VPSRLD     $0x14, Y9, Y9
	VPXOR      Y15, Y9, Y9
	VPSLLD     $0x0c, Y10, Y15
	VPSRLD     $0x14, Y10, Y10
	VPXOR      Y15, Y10, Y10
	VPSLLD     $0x0c, Y11, Y15
	VPSRLD     $0x14, Y11, Y11
	VPXOR      Y15, Y11, Y11
	VMOVDQA    224(BP), Y15
	ADDQ       (DI), R10
	ADCQ       8(DI), R11
	ADCQ       $0x01, R12
	MOVQ       (BP), DX
	MOVQ       DX, R15
	MULXQ      R10, R13, R14
	IMULQ      R12, R15
	MULXQ      R11, AX, DX
	ADDQ       AX, R14
	ADCQ       DX, R15
	MOVQ       8(BP), DX
	MULXQ      R10, R10, AX
	ADDQ       R10, R14
	MULXQ      R11, R11, R8
	ADCQ       R11, R15
	ADCQ       $0x00, R8
	IMULQ      R12, DX
	ADDQ       AX, R15
	ADCQ       DX, R8
	MOVQ       R13, R10
	MOVQ       R14, R11
	MOVQ       R15, R12
	ANDQ       $0x03, R12
	MOVQ       R15, R13
	ANDQ       $-4, R13
	MOVQ       R8, R14
	SHRQ       $0x02, R8, R15
	SHRQ       $0x02, R8
	ADDQ       R13, R10
	ADCQ       R14, R11
	ADCQ       $0x00, R12
	ADDQ       R15, R10
	ADCQ       R8, R11
	ADCQ       $0x00, R12
	VPADDD     Y14, Y0, Y0
	VPADDD     Y9, Y5, Y5
	VPADDD     Y10, Y6, Y6
	VPADDD     Y11, Y7, Y7
	VPXOR      Y0, Y4, Y4
	VPXOR      Y5, Y1, Y1
	VPXOR      Y6, Y2, Y2
	VPXOR      Y7, Y3, Y3
	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
	VPSHUFB    ·rol8<>+0(SB), Y3, Y3
	VPADDD     Y4, Y12, Y12
	VPADDD     Y1, Y13, Y13
	VPADDD     Y2, Y8, Y8
	VPADDD     Y3, Y15, Y15
	VPXOR      Y12, Y14, Y14
	VPXOR      Y13, Y9, Y9
	VPXOR      Y8, Y10, Y10
	VPXOR      Y15, Y11, Y11
	VMOVDQA    Y15, 224(BP)
	VPSLLD     $0x07, Y14, Y15
	VPSRLD     $0x19, Y14, Y14
	VPXOR      Y15, Y14, Y14
	VPSLLD     $0x07, Y9, Y15
	VPSRLD     $0x19, Y9, Y9
	VPXOR      Y15, Y9, Y9
	VPSLLD     $0x07, Y10, Y15
	VPSRLD     $0x19, Y10, Y10
	VPXOR      Y15, Y10, Y10
	VPSLLD     $0x07, Y11, Y15
	VPSRLD     $0x19, Y11, Y11
	VPXOR      Y15, Y11, Y11
	VMOVDQA    224(BP), Y15
	VPALIGNR   $0x04, Y14, Y14, Y14
	VPALIGNR   $0x04, Y9, Y9, Y9
	VPALIGNR   $0x04, Y10, Y10, Y10
	VPALIGNR   $0x04, Y11, Y11, Y11
	VPALIGNR   $0x08, Y12, Y12, Y12
	VPALIGNR   $0x08, Y13, Y13, Y13
	VPALIGNR   $0x08, Y8, Y8, Y8
	VPALIGNR   $0x08, Y15, Y15, Y15
	VPALIGNR   $0x0c, Y4, Y4, Y4
	VPALIGNR   $0x0c, Y1, Y1, Y1
	VPALIGNR   $0x0c, Y2, Y2, Y2
	VPALIGNR   $0x0c, Y3, Y3, Y3
	VPADDD     Y14, Y0, Y0
	VPADDD     Y9, Y5, Y5
	VPADDD     Y10, Y6, Y6
	VPADDD     Y11, Y7, Y7
	VPXOR      Y0, Y4, Y4
	VPXOR      Y5, Y1, Y1
	VPXOR      Y6, Y2, Y2
	VPXOR      Y7, Y3, Y3
	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
	VPSHUFB    ·rol16<>+0(SB), Y3, Y3
	VPADDD     Y4, Y12, Y12
	VPADDD     Y1, Y13, Y13
	VPADDD     Y2, Y8, Y8
	VPADDD     Y3, Y15, Y15
	VPXOR      Y12, Y14, Y14
	VPXOR      Y13, Y9, Y9
	VPXOR      Y8, Y10, Y10
	VPXOR      Y15, Y11, Y11
	ADDQ       16(DI), R10
	ADCQ       24(DI), R11
	ADCQ       $0x01, R12
	MOVQ       (BP), DX
	MOVQ       DX, R15
	MULXQ      R10, R13, R14
	IMULQ      R12, R15
	MULXQ      R11, AX, DX
	ADDQ       AX, R14
	ADCQ       DX, R15
	MOVQ       8(BP), DX
	MULXQ      R10, R10, AX
	ADDQ       R10, R14
	MULXQ      R11, R11, R8
	ADCQ       R11, R15
	ADCQ       $0x00, R8
	IMULQ      R12, DX
	ADDQ       AX, R15
	ADCQ       DX, R8
	MOVQ       R13, R10
	MOVQ       R14, R11
	MOVQ       R15, R12
	ANDQ       $0x03, R12
	MOVQ       R15, R13
	ANDQ       $-4, R13
	MOVQ       R8, R14
	SHRQ       $0x02, R8, R15
	SHRQ       $0x02, R8
	ADDQ       R13, R10
	ADCQ       R14, R11
	ADCQ       $0x00, R12
	ADDQ       R15, R10
	ADCQ       R8, R11
	ADCQ       $0x00, R12
	LEAQ       32(DI), DI
	VMOVDQA    Y15, 224(BP)
	VPSLLD     $0x0c, Y14, Y15
	VPSRLD     $0x14, Y14, Y14
	VPXOR      Y15, Y14, Y14
	VPSLLD     $0x0c, Y9, Y15
	VPSRLD     $0x14, Y9, Y9
	VPXOR      Y15, Y9, Y9
	VPSLLD     $0x0c, Y10, Y15
	VPSRLD     $0x14, Y10, Y10
	VPXOR      Y15, Y10, Y10
	VPSLLD     $0x0c, Y11, Y15
	VPSRLD     $0x14, Y11, Y11
	VPXOR      Y15, Y11, Y11
	VMOVDQA    224(BP), Y15
	VPADDD     Y14, Y0, Y0
	VPADDD     Y9, Y5, Y5
	VPADDD     Y10, Y6, Y6
	VPADDD     Y11, Y7, Y7
	VPXOR      Y0, Y4, Y4
	VPXOR      Y5, Y1, Y1
	VPXOR      Y6, Y2, Y2
	VPXOR      Y7, Y3, Y3
	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
	VPSHUFB    ·rol8<>+0(SB), Y3, Y3
	VPADDD     Y4, Y12, Y12
	VPADDD     Y1, Y13, Y13
	VPADDD     Y2, Y8, Y8
	VPADDD     Y3, Y15, Y15
	VPXOR      Y12, Y14, Y14
	VPXOR      Y13, Y9, Y9
	VPXOR      Y8, Y10, Y10
	VPXOR      Y15, Y11, Y11
	VMOVDQA    Y15, 224(BP)
	VPSLLD     $0x07, Y14, Y15
	VPSRLD     $0x19, Y14, Y14
	VPXOR      Y15, Y14, Y14
	VPSLLD     $0x07, Y9, Y15
	VPSRLD     $0x19, Y9, Y9
	VPXOR      Y15, Y9, Y9
	VPSLLD     $0x07, Y10, Y15
	VPSRLD     $0x19, Y10, Y10
	VPXOR      Y15, Y10, Y10
	VPSLLD     $0x07, Y11, Y15
	VPSRLD     $0x19, Y11, Y11
	VPXOR      Y15, Y11, Y11
	VMOVDQA    224(BP), Y15
	VPALIGNR   $0x0c, Y14, Y14, Y14
	VPALIGNR   $0x0c, Y9, Y9, Y9
	VPALIGNR   $0x0c, Y10, Y10, Y10
	VPALIGNR   $0x0c, Y11, Y11, Y11
	VPALIGNR   $0x08, Y12, Y12, Y12
	VPALIGNR   $0x08, Y13, Y13, Y13
	VPALIGNR   $0x08, Y8, Y8, Y8
	VPALIGNR   $0x08, Y15, Y15, Y15
	VPALIGNR   $0x04, Y4, Y4, Y4
	VPALIGNR   $0x04, Y1, Y1, Y1
	VPALIGNR   $0x04, Y2, Y2, Y2
	VPALIGNR   $0x04, Y3, Y3, Y3
	DECQ       CX
	JG         sealAVX2Tail512LoopA
	DECQ       R9
	JGE        sealAVX2Tail512LoopB
	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
	VPADDD     ·chacha20Constants<>+0(SB), Y7, Y7
	VPADDD     32(BP), Y14, Y14
	VPADDD     32(BP), Y9, Y9
	VPADDD     32(BP), Y10, Y10
	VPADDD     32(BP), Y11, Y11
	VPADDD     64(BP), Y12, Y12
	VPADDD     64(BP), Y13, Y13
	VPADDD     64(BP), Y8, Y8
	VPADDD     64(BP), Y15, Y15
	VPADDD     96(BP), Y4, Y4
	VPADDD     128(BP), Y1, Y1
	VPADDD     160(BP), Y2, Y2
	VPADDD     192(BP), Y3, Y3
	VMOVDQA    Y15, 224(BP)
	VPERM2I128 $0x02, Y0, Y14, Y15
	VPXOR      (SI), Y15, Y15
	VMOVDQU    Y15, (DI)
	VPERM2I128 $0x02, Y12, Y4, Y15
	VPXOR      32(SI), Y15, Y15
	VMOVDQU    Y15, 32(DI)
	VPERM2I128 $0x13, Y0, Y14, Y15
	VPXOR      64(SI), Y15, Y15
	VMOVDQU    Y15, 64(DI)
	VPERM2I128 $0x13, Y12, Y4, Y15
	VPXOR      96(SI), Y15, Y15
	VMOVDQU    Y15, 96(DI)
	VPERM2I128 $0x02, Y5, Y9, Y0
	VPERM2I128 $0x02, Y13, Y1, Y14
	VPERM2I128 $0x13, Y5, Y9, Y12
	VPERM2I128 $0x13, Y13, Y1, Y4
	VPXOR      128(SI), Y0, Y0
	VPXOR      160(SI), Y14, Y14
	VPXOR      192(SI), Y12, Y12
	VPXOR      224(SI), Y4, Y4
	VMOVDQU    Y0, 128(DI)
	VMOVDQU    Y14, 160(DI)
	VMOVDQU    Y12, 192(DI)
	VMOVDQU    Y4, 224(DI)
	VPERM2I128 $0x02, Y6, Y10, Y0
	VPERM2I128 $0x02, Y8, Y2, Y14
	VPERM2I128 $0x13, Y6, Y10, Y12
	VPERM2I128 $0x13, Y8, Y2, Y4
	VPXOR      256(SI), Y0, Y0
	VPXOR      288(SI), Y14, Y14
	VPXOR      320(SI), Y12, Y12
	VPXOR      352(SI), Y4, Y4
	VMOVDQU    Y0, 256(DI)
	VMOVDQU    Y14, 288(DI)
	VMOVDQU    Y12, 320(DI)
	VMOVDQU    Y4, 352(DI)
	MOVQ       $0x00000180, CX
	LEAQ       384(SI), SI
	SUBQ       $0x00000180, BX
	VPERM2I128 $0x02, Y7, Y11, Y0
	VPERM2I128 $0x02, 224(BP), Y3, Y14
	VPERM2I128 $0x13, Y7, Y11, Y12
	VPERM2I128 $0x13, 224(BP), Y3, Y4
	JMP        sealAVX2SealHash