// Copyright 2024 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This assembly implementation was originally from https://golang.org/cl/24717 by Vlad Krasnov of CloudFlare.

package main

import (
	"fmt"
	"os"
	"strings"

	. "github.com/mmcloughlin/avo/build"
	"github.com/mmcloughlin/avo/ir"
	. "github.com/mmcloughlin/avo/operand"
	. "github.com/mmcloughlin/avo/reg"
	_ "golang.org/x/crypto/chacha20poly1305"
)

//go:generate go run . -out ../chacha20poly1305_amd64.s -pkg chacha20poly1305

var (
	// General register allocation
	oup  GPPhysical = RDI
	inp             = RSI
	inl             = RBX
	adp             = RCX // free to reuse, after we hash the additional data
	keyp            = R8  // free to reuse, when we copy the key to stack
	itr2            = R9  // general iterator
	itr1            = RCX // general iterator
	acc0            = R10
	acc1            = R11
	acc2            = R12
	t0              = R13
	t1              = R14
	t2              = R15
	t3              = R8

	// Register and stack allocation for the SSE code
	rStore      Mem         = Mem{Base: BP}.Offset(0 * 16)
	sStore                  = Mem{Base: BP}.Offset(1 * 16)
	state1Store             = Mem{Base: BP}.Offset(2 * 16)
	state2Store             = Mem{Base: BP}.Offset(3 * 16)
	tmpStore                = Mem{Base: BP}.Offset(4 * 16)
	ctr0Store               = Mem{Base: BP}.Offset(5 * 16)
	ctr1Store               = Mem{Base: BP}.Offset(6 * 16)
	ctr2Store               = Mem{Base: BP}.Offset(7 * 16)
	ctr3Store               = Mem{Base: BP}.Offset(8 * 16)
	A0          VecPhysical = X0
	A1                      = X1
	A2                      = X2
	B0                      = X3
	B1                      = X4
	B2                      = X5
	C0                      = X6
	C1                      = X7
	C2                      = X8
	D0                      = X9
	D1                      = X10
	D2                      = X11
	T0                      = X12
	T1                      = X13
	T2                      = X14
	T3                      = X15
	A3                      = T0
	B3                      = T1
	C3                      = T2
	D3                      = T3

	// Register and stack allocation for the AVX2 code
	rsStoreAVX2     Mem         = Mem{Base: BP}.Offset(0 * 32)
	state1StoreAVX2             = Mem{Base: BP}.Offset(1 * 32)
	state2StoreAVX2             = Mem{Base: BP}.Offset(2 * 32)
	ctr0StoreAVX2               = Mem{Base: BP}.Offset(3 * 32)
	ctr1StoreAVX2               = Mem{Base: BP}.Offset(4 * 32)
	ctr2StoreAVX2               = Mem{Base: BP}.Offset(5 * 32)
	ctr3StoreAVX2               = Mem{Base: BP}.Offset(6 * 32)
	tmpStoreAVX2                = Mem{Base: BP}.Offset(7 * 32) // 256 bytes on stack
	AA0             VecPhysical = Y0
	AA1                         = Y5
	AA2                         = Y6
	AA3                         = Y7
	BB0                         = Y14
	BB1                         = Y9
	BB2                         = Y10
	BB3                         = Y11
	CC0                         = Y12
	CC1                         = Y13
	CC2                         = Y8
	CC3                         = Y15
	DD0                         = Y4
	DD1                         = Y1
	DD2                         = Y2
	DD3                         = Y3
	TT0                         = DD3
	TT1                         = AA3
	TT2                         = BB3
	TT3                         = CC3
)

const ThatPeskyUnicodeDot = "\u00b7"

func main() {
	Package("golang.org/x/crypto/chacha20poly1305")
	ConstraintExpr("gc,!purego")
	polyHashADInternal()
	chacha20Poly1305Open()
	chacha20Poly1305Seal()
	Generate()

	var internalFunctions []string = []string{"·polyHashADInternal"}
	removePeskyUnicodeDot(internalFunctions, "../chacha20poly1305_amd64.s")
}

// Utility function to emit BYTE instruction
func BYTE(u8 U8) {
	Instruction(&ir.Instruction{Opcode: "BYTE", Operands: []Op{u8}})
}

// PALIGNR $4, X3, X3
func shiftB0Left() {
	BYTE(U8(0x66))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xdb))
	BYTE(U8(0x04))
}

// PALIGNR $4, X4, X4
func shiftB1Left() {
	BYTE(U8(0x66))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xe4))
	BYTE(U8(0x04))
}

// PALIGNR $4, X5, X5
func shiftB2Left() {
	BYTE(U8(0x66))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xed))
	BYTE(U8(0x04))
}

// PALIGNR $4, X13, X13
func shiftB3Left() {
	BYTE(U8(0x66))
	BYTE(U8(0x45))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xed))
	BYTE(U8(0x04))
}

// PALIGNR $8, X6, X6
func shiftC0Left() {
	BYTE(U8(0x66))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xf6))
	BYTE(U8(0x08))
}

// PALIGNR $8, X7, X7
func shiftC1Left() {
	BYTE(U8(0x66))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xff))
	BYTE(U8(0x08))
}

// PALIGNR $8, X8, X8
func shiftC2Left() {
	BYTE(U8(0x66))
	BYTE(U8(0x45))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xc0))
	BYTE(U8(0x08))
}

// PALIGNR $8, X14, X14
func shiftC3Left() {
	BYTE(U8(0x66))
	BYTE(U8(0x45))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xf6))
	BYTE(U8(0x08))
}

// PALIGNR $12, X9, X9
func shiftD0Left() {
	BYTE(U8(0x66))
	BYTE(U8(0x45))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xc9))
	BYTE(U8(0x0c))
}

// PALIGNR $12, X10, X10
func shiftD1Left() {
	BYTE(U8(0x66))
	BYTE(U8(0x45))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xd2))
	BYTE(U8(0x0c))
}

// PALIGNR $12, X11, X11
func shiftD2Left() {
	BYTE(U8(0x66))
	BYTE(U8(0x45))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xdb))
	BYTE(U8(0x0c))
}

// PALIGNR $12, X15, X15
func shiftD3Left() {
	BYTE(U8(0x66))
	BYTE(U8(0x45))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xff))
	BYTE(U8(0x0c))
}

// PALIGNR $12, X3, X3
func shiftB0Right() {
	BYTE(U8(0x66))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xdb))
	BYTE(U8(0x0c))
}

// PALIGNR $12, X4, X4
func shiftB1Right() {
	BYTE(U8(0x66))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xe4))
	BYTE(U8(0x0c))
}

// PALIGNR $12, X5, X5
func shiftB2Right() {
	BYTE(U8(0x66))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xed))
	BYTE(U8(0x0c))
}

// PALIGNR $12, X13, X13
func shiftB3Right() {
	BYTE(U8(0x66))
	BYTE(U8(0x45))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xed))
	BYTE(U8(0x0c))
}

func shiftC0Right() {
	shiftC0Left()
}

func shiftC1Right() {
	shiftC1Left()
}

func shiftC2Right() {
	shiftC2Left()
}

func shiftC3Right() {
	shiftC3Left()
}

// PALIGNR $4, X9, X9
func shiftD0Right() {
	BYTE(U8(0x66))
	BYTE(U8(0x45))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xc9))
	BYTE(U8(0x04))
}

// PALIGNR $4, X10, X10
func shiftD1Right() {
	BYTE(U8(0x66))
	BYTE(U8(0x45))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xd2))
	BYTE(U8(0x04))
}

// PALIGNR $4, X11, X11
func shiftD2Right() {
	BYTE(U8(0x66))
	BYTE(U8(0x45))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xdb))
	BYTE(U8(0x04))
}

// PALIGNR $4, X15, X15
func shiftD3Right() {
	BYTE(U8(0x66))
	BYTE(U8(0x45))
	BYTE(U8(0x0f))
	BYTE(U8(0x3a))
	BYTE(U8(0x0f))
	BYTE(U8(0xff))
	BYTE(U8(0x04))
}

// ##~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~SOME  MACROS~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~##

// Hack: ROL must be a #define macro as it is referenced by other macros
func defineROL() {
	definition :=
		`#define ROL(N, R, T) \
		MOVO R, T; \
		PSLLL $(N), T; \
		PSRLL $(32-(N)), R; \
		PXOR T, R`
	Comment("ROL rotates the uint32s in register R left by N bits, using temporary T.")
	Instruction(&ir.Instruction{Opcode: definition})
}

// ROL rotates the uint32s in register R left by N bits, using temporary T.
func ROL(N uint64, R, T VecPhysical) {
	// Hack: ROL must be a #define macro as it is referenced by other macros
	Instruction(&ir.Instruction{Opcode: fmt.Sprintf("ROL(%s, %s, %s)", I8(N).Asm(), R.Asm(), T.Asm())})
}

// Hack to get Avo to generate an #ifdef
//
// ROL16(R, T) definition depends on a compiler flag that specifies amd64 architectural level.
func defineROL16() {
	definition :=
		`#ifdef GOAMD64_v2
		#define ROL16(R, T) PSHUFB ·rol16<>(SB), R
	#else
		#define ROL16(R, T) ROL(16, R, T)
	#endif`

	Comment("ROL16 rotates the uint32s in register R left by 16, using temporary T if needed.")
	Instruction(&ir.Instruction{Opcode: definition})
}

// Hack to emit macro call
//
// ROL16 rotates the uint32s in register R left by 16, using temporary T if needed.
func ROL16(R, T VecPhysical) {
	Instruction(&ir.Instruction{Opcode: fmt.Sprintf("ROL16(%s, %s)", R.Asm(), T.Asm())})
}

// Hack to get Avo to generate an #ifdef
//
// ROL8(R, T) definition depends on a compiler flag that specifies amd64 architectural level.
func defineROL8() {
	definition :=
		`#ifdef GOAMD64_v2
		#define ROL8(R, T) PSHUFB ·rol8<>(SB), R
	#else
		#define ROL8(R, T) ROL(8, R, T)
	#endif`

	Comment("ROL8 rotates the uint32s in register R left by 8, using temporary T if needed.")
	Instruction(&ir.Instruction{Opcode: definition})
}

// Hack to emit macro call
//
// ROL8 rotates the uint32s in register R left by 8, using temporary T if needed.
func ROL8(R, T VecPhysical) {
	Instruction(&ir.Instruction{Opcode: fmt.Sprintf("ROL8(%s, %s)", R.Asm(), T.Asm())})
}

func chachaQR(A, B, C, D, T VecPhysical) {
	PADDD(B, A)
	PXOR(A, D)
	ROL16(D, T)
	PADDD(D, C)
	PXOR(C, B)
	MOVO(B, T)
	PSLLL(Imm(12), T)
	PSRLL(Imm(20), B)
	PXOR(T, B)
	PADDD(B, A)
	PXOR(A, D)
	ROL8(D, T)
	PADDD(D, C)
	PXOR(C, B)
	MOVO(B, T)
	PSLLL(Imm(7), T)
	PSRLL(Imm(25), B)
	PXOR(T, B)
}

func chachaQR_AVX2(A, B, C, D, T VecPhysical) {
	VPADDD(B, A, A)
	VPXOR(A, D, D)
	rol16 := rol16_DATA()
	VPSHUFB(rol16, D, D)
	VPADDD(D, C, C)
	VPXOR(C, B, B)
	VPSLLD(Imm(12), B, T)
	VPSRLD(Imm(20), B, B)
	VPXOR(T, B, B)
	VPADDD(B, A, A)
	VPXOR(A, D, D)
	rol8 := rol8_DATA()
	VPSHUFB(rol8, D, D)
	VPADDD(D, C, C)
	VPXOR(C, B, B)
	VPSLLD(Imm(7), B, T)
	VPSRLD(Imm(25), B, B)
	VPXOR(T, B, B)
}

func polyAdd(S Mem) {
	ADDQ(S, acc0)
	ADCQ(S.Offset(8), acc1)
	ADCQ(Imm(1), acc2)
}

func polyMulStage1() {
	MOVQ(Mem{Base: BP}.Offset(0*8), RAX)
	MOVQ(RAX, t2)
	MULQ(acc0)
	MOVQ(RAX, t0)
	MOVQ(RDX, t1)
	MOVQ(Mem{Base: BP}.Offset(0*8), RAX)
	MULQ(acc1)
	IMULQ(acc2, t2)
	ADDQ(RAX, t1)
	ADCQ(RDX, t2)
}

func polyMulStage2() {
	MOVQ(Mem{Base: BP}.Offset(1*8), RAX)
	MOVQ(RAX, t3)
	MULQ(acc0)
	ADDQ(RAX, t1)
	ADCQ(Imm(0), RDX)
	MOVQ(RDX, acc0)
	MOVQ(Mem{Base: BP}.Offset(1*8), RAX)
	MULQ(acc1)
	ADDQ(RAX, t2)
	ADCQ(Imm(0), RDX)
}

func polyMulStage3() {
	IMULQ(acc2, t3)
	ADDQ(acc0, t2)
	ADCQ(RDX, t3)
}

func polyMulReduceStage() {
	MOVQ(t0, acc0)
	MOVQ(t1, acc1)
	MOVQ(t2, acc2)
	ANDQ(Imm(3), acc2)
	MOVQ(t2, t0)
	ANDQ(I8(-4), t0)
	MOVQ(t3, t1)
	SHRQ(Imm(2), t3, t2)
	SHRQ(Imm(2), t3)
	ADDQ(t0, acc0)
	ADCQ(t1, acc1)
	ADCQ(Imm(0), acc2)
	ADDQ(t2, acc0)
	ADCQ(t3, acc1)
	ADCQ(Imm(0), acc2)
}

func polyMulStage1_AVX2() {
	MOVQ(Mem{Base: BP}.Offset(0*8), RDX)
	MOVQ(RDX, t2)
	MULXQ(acc0, t0, t1)
	IMULQ(acc2, t2)
	MULXQ(acc1, RAX, RDX)
	ADDQ(RAX, t1)
	ADCQ(RDX, t2)
}

func polyMulStage2_AVX2() {
	MOVQ(Mem{Base: BP}.Offset(1*8), RDX)
	MULXQ(acc0, acc0, RAX)
	ADDQ(acc0, t1)
	MULXQ(acc1, acc1, t3)
	ADCQ(acc1, t2)
	ADCQ(Imm(0), t3)
}

func polyMulStage3_AVX2() {
	IMULQ(acc2, RDX)
	ADDQ(RAX, t2)
	ADCQ(RDX, t3)
}

func polyMul() {
	polyMulStage1()
	polyMulStage2()
	polyMulStage3()
	polyMulReduceStage()
}

func polyMulAVX2() {
	polyMulStage1_AVX2()
	polyMulStage2_AVX2()
	polyMulStage3_AVX2()
	polyMulReduceStage()
}

// ----------------------------------------------------------------------------
// ----------------------------------------------------------------------------

func polyHashADInternal() {
	Function("polyHashADInternal<>")
	Attributes(NOSPLIT)
	AllocLocal(0)

	Comment("Hack: Must declare #define macros inside of a function due to Avo constraints")
	defineROL()
	defineROL8()
	defineROL16()

	// adp points to beginning of additional data
	// itr2 holds ad length
	XORQ(acc0, acc0)
	XORQ(acc1, acc1)
	XORQ(acc2, acc2)
	CMPQ(itr2, Imm(13))
	JNE(LabelRef("hashADLoop"))

	openFastTLSAD()
	hashADLoop()
	hashADTail()
	hashADTailLoop()
	hashADTailFinish()
	hashADDone()
}

// Special treatment for the TLS case of 13 bytes
func openFastTLSAD() {
	Label("openFastTLSAD")
	MOVQ(Mem{Base: adp}, acc0)
	MOVQ(Mem{Base: adp}.Offset(5), acc1)
	SHRQ(Imm(24), acc1)
	MOVQ(U32(1), acc2)
	polyMul()
	RET()
}

// Hash in 16 byte chunks
func hashADLoop() {
	Label("hashADLoop")
	Comment("Hash in 16 byte chunks")
	CMPQ(itr2, Imm(16))
	JB(LabelRef("hashADTail"))
	polyAdd(Mem{Base: adp}.Offset(0))
	LEAQ(Mem{Base: adp}.Offset(1*16), adp)
	SUBQ(Imm(16), itr2)
	polyMul()
	JMP(LabelRef("hashADLoop"))
}

func hashADTail() {
	Label("hashADTail")
	CMPQ(itr2, Imm(0))
	JE(LabelRef("hashADDone"))

	Comment("Hash last < 16 byte tail")
	XORQ(t0, t0)
	XORQ(t1, t1)
	XORQ(t2, t2)
	ADDQ(itr2, adp)
}

func hashADTailLoop() {
	Label("hashADTailLoop")
	SHLQ(Imm(8), t0, t1)
	SHLQ(Imm(8), t0)
	// Hack to get Avo to emit:
	// 	MOVB -1(adp), t2
	Instruction(&ir.Instruction{Opcode: "MOVB", Operands: []Op{Mem{Base: adp}.Offset(-1), t2}})
	XORQ(t2, t0)
	DECQ(adp)
	DECQ(itr2)
	JNE(LabelRef("hashADTailLoop"))
}

func hashADTailFinish() {
	ADDQ(t0, acc0)
	ADCQ(t1, acc1)
	ADCQ(Imm(1), acc2)
	polyMul()
}

// Finished AD
func hashADDone() {
	Label("hashADDone")
	RET()
}

// ----------------------------------------------------------------------------
// ----------------------------------------------------------------------------

// Implements the following function signature:
//
//	func chacha20Poly1305Open(dst []byte, key []uint32, src []byte, ad []byte) bool
func chacha20Poly1305Open() {
	Implement("chacha20Poly1305Open")
	Attributes(0)
	AllocLocal(288)

	Comment("For aligned stack access")
	MOVQ(RSP, RBP)
	ADDQ(Imm(32), RBP)
	ANDQ(I8(-32), RBP)

	Load(Param("dst").Base(), oup)
	Load(Param("key").Base(), keyp)
	Load(Param("src").Base(), inp)
	Load(Param("src").Len(), inl)
	Load(Param("ad").Base(), adp)

	Comment("Check for AVX2 support")
	CMPB(Mem{Symbol: Symbol{Name: ThatPeskyUnicodeDot + "useAVX2"}, Base: StaticBase}, Imm(1))
	JE(LabelRef("chacha20Poly1305Open_AVX2"))

	Comment("Special optimization, for very short buffers")
	CMPQ(inl, Imm(128))
	JBE(LabelRef("openSSE128")) // About 16% faster

	Comment("For long buffers, prepare the poly key first")
	chacha20Constants := chacha20Constants_DATA()
	MOVOU(chacha20Constants, A0)
	MOVOU(Mem{Base: keyp}.Offset(1*16), B0)
	MOVOU(Mem{Base: keyp}.Offset(2*16), C0)
	MOVOU(Mem{Base: keyp}.Offset(3*16), D0)
	MOVO(D0, T1)

	Comment("Store state on stack for future use")
	MOVO(B0, state1Store)
	MOVO(C0, state2Store)
	MOVO(D0, ctr3Store)
	MOVQ(U32(10), itr2)

	openSSEPreparePolyKey()
	openSSEMainLoop()
	openSSEInternalLoop()
	openSSEMainLoopDone()
	openSSEFinalize()

	// ----------------------------------------------------------------------------
	// Special optimization for buffers smaller than 129 bytes
	openSSE128()
	openSSE128InnerCipherLoop()
	openSSE128Open()
	openSSETail16()
	openSSETail16Store()

	// ----------------------------------------------------------------------------
	// Special optimization for the last 64 bytes of ciphertext
	openSSETail64()
	openSSETail64LoopA()
	openSSETail64LoopB()
	openSSETail64DecLoop()
	openSSETail64DecLoopDone()

	// ----------------------------------------------------------------------------
	// Special optimization for the last 128 bytes of ciphertext
	openSSETail128()
	openSSETail128LoopA()
	openSSETail128LoopB()

	// ----------------------------------------------------------------------------
	// Special optimization for the last 192 bytes of ciphertext
	openSSETail192()
	openSSLTail192LoopA()
	openSSLTail192LoopB()
	openSSLTail192Store()

	// ----------------------------------------------------------------------------
	// Special optimization for the last 256 bytes of ciphertext
	openSSETail256()
	openSSETail256Loop()
	openSSETail256HashLoop()

	// ----------------------------------------------------------------------------
	// ------------------------- AVX2 Code ----------------------------------------
	chacha20Poly1305Open_AVX2()
	openAVX2PreparePolyKey()
	openAVX2InitialHash64()
	openAVX2MainLoop()
	openAVX2InternalLoop()
	openAVX2MainLoopDone()

	// ----------------------------------------------------------------------------
	// Special optimization for buffers smaller than 193 bytes
	openAVX2192()
	openAVX2192InnerCipherLoop()
	openAVX2ShortOpen()
	openAVX2ShortOpenLoop()
	openAVX2ShortTail32()
	openAVX2ShortDone()

	// ----------------------------------------------------------------------------
	// Special optimization for buffers smaller than 321 bytes
	openAVX2320()
	openAVX2320InnerCipherLoop()

	// ----------------------------------------------------------------------------
	// Special optimization for the last 128 bytes of ciphertext
	openAVX2Tail128()
	openAVX2Tail128LoopA()
	openAVX2Tail128LoopB()
	openAVX2TailLoop()
	openAVX2Tail()
	openAVX2TailDone()

	// ----------------------------------------------------------------------------
	// Special optimization for the last 256 bytes of ciphertext
	openAVX2Tail256()
	openAVX2Tail256LoopA()
	openAVX2Tail256LoopB()
	openAVX2Tail256Hash()
	openAVX2Tail256HashEnd()

	// ----------------------------------------------------------------------------
	// Special optimization for the last 384 bytes of ciphertext
	openAVX2Tail384()
	openAVX2Tail384LoopB()
	openAVX2Tail384LoopA()
	openAVX2Tail384Hash()
	openAVX2Tail384HashEnd()

	// ----------------------------------------------------------------------------
	// Special optimization for the last 512 bytes of ciphertext
	openAVX2Tail512()
	openAVX2Tail512LoopB()
	openAVX2Tail512LoopA()
	openAVX2Tail512HashLoop()
	openAVX2Tail512HashEnd()
}

func openSSEPreparePolyKey() {
	Label("openSSEPreparePolyKey")
	chachaQR(A0, B0, C0, D0, T0)
	shiftB0Left()
	shiftC0Left()
	shiftD0Left()
	chachaQR(A0, B0, C0, D0, T0)
	shiftB0Right()
	shiftC0Right()
	shiftD0Right()
	DECQ(itr2)
	JNE(LabelRef("openSSEPreparePolyKey"))

	Comment("A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded")
	chacha20Constants := chacha20Constants_DATA()
	PADDL(chacha20Constants, A0)
	PADDL(state1Store, B0)

	Comment("Clamp and store the key")
	polyClampMask := polyClampMask_DATA()
	PAND(polyClampMask, A0)
	MOVO(A0, rStore)
	MOVO(B0, sStore)

	Comment("Hash AAD")
	Load(Param("ad").Len(), itr2)
	CALL(LabelRef("polyHashADInternal<>(SB)"))
}

func openSSEMainLoop() {
	Label("openSSEMainLoop")
	CMPQ(inl, U32(256))
	JB(LabelRef("openSSEMainLoopDone"))

	chacha20Constants := chacha20Constants_DATA()
	sseIncMask := sseIncMask_DATA()

	Comment("Load state, increment counter blocks")
	MOVO(chacha20Constants, A0)
	MOVO(state1Store, B0)
	MOVO(state2Store, C0)
	MOVO(ctr3Store, D0)
	PADDL(sseIncMask, D0)
	MOVO(A0, A1)
	MOVO(B0, B1)
	MOVO(C0, C1)
	MOVO(D0, D1)
	PADDL(sseIncMask, D1)
	MOVO(A1, A2)
	MOVO(B1, B2)
	MOVO(C1, C2)
	MOVO(D1, D2)
	PADDL(sseIncMask, D2)
	MOVO(A2, A3)
	MOVO(B2, B3)
	MOVO(C2, C3)
	MOVO(D2, D3)
	PADDL(sseIncMask, D3)

	Comment("Store counters")
	MOVO(D0, ctr0Store)
	MOVO(D1, ctr1Store)
	MOVO(D2, ctr2Store)
	MOVO(D3, ctr3Store)

	Comment("There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we hash")
	Comment("2 blocks, and for the remaining 4 only 1 block - for a total of 16")
	MOVQ(U32(4), itr1)
	MOVQ(inp, itr2)
}

func openSSEInternalLoop() {
	Label("openSSEInternalLoop")
	MOVO(C3, tmpStore)
	chachaQR(A0, B0, C0, D0, C3)
	chachaQR(A1, B1, C1, D1, C3)
	chachaQR(A2, B2, C2, D2, C3)
	MOVO(tmpStore, C3)
	MOVO(C1, tmpStore)
	chachaQR(A3, B3, C3, D3, C1)
	MOVO(tmpStore, C1)
	polyAdd(Mem{Base: itr2}.Offset(0))
	shiftB0Left()
	shiftB1Left()
	shiftB2Left()
	shiftB3Left()
	shiftC0Left()
	shiftC1Left()
	shiftC2Left()
	shiftC3Left()
	shiftD0Left()
	shiftD1Left()
	shiftD2Left()
	shiftD3Left()
	polyMulStage1()
	polyMulStage2()
	LEAQ(Mem{Base: itr2}.Offset(2*8), itr2)
	MOVO(C3, tmpStore)
	chachaQR(A0, B0, C0, D0, C3)
	chachaQR(A1, B1, C1, D1, C3)
	chachaQR(A2, B2, C2, D2, C3)
	MOVO(tmpStore, C3)
	MOVO(C1, tmpStore)
	polyMulStage3()
	chachaQR(A3, B3, C3, D3, C1)
	MOVO(tmpStore, C1)
	polyMulReduceStage()
	shiftB0Right()
	shiftB1Right()
	shiftB2Right()
	shiftB3Right()
	shiftC0Right()
	shiftC1Right()
	shiftC2Right()
	shiftC3Right()
	shiftD0Right()
	shiftD1Right()
	shiftD2Right()
	shiftD3Right()
	DECQ(itr1)
	JGE(LabelRef("openSSEInternalLoop"))

	polyAdd(Mem{Base: itr2}.Offset(0))
	polyMul()
	LEAQ(Mem{Base: itr2}.Offset(2*8), itr2)

	CMPQ(itr1, I8(-6))
	JG(LabelRef("openSSEInternalLoop"))

	chacha20Constants := chacha20Constants_DATA()
	Comment("Add in the state")
	PADDD(chacha20Constants, A0)
	PADDD(chacha20Constants, A1)
	PADDD(chacha20Constants, A2)
	PADDD(chacha20Constants, A3)
	PADDD(state1Store, B0)
	PADDD(state1Store, B1)
	PADDD(state1Store, B2)
	PADDD(state1Store, B3)
	PADDD(state2Store, C0)
	PADDD(state2Store, C1)
	PADDD(state2Store, C2)
	PADDD(state2Store, C3)
	PADDD(ctr0Store, D0)
	PADDD(ctr1Store, D1)
	PADDD(ctr2Store, D2)
	PADDD(ctr3Store, D3)

	Comment("Load - xor - store")
	MOVO(D3, tmpStore)
	MOVOU(Mem{Base: inp}.Offset(0*16), D3)
	PXOR(D3, A0)
	MOVOU(A0, Mem{Base: oup}.Offset(0*16))
	MOVOU(Mem{Base: inp}.Offset(1*16), D3)
	PXOR(D3, B0)
	MOVOU(B0, Mem{Base: oup}.Offset(1*16))
	MOVOU(Mem{Base: inp}.Offset(2*16), D3)
	PXOR(D3, C0)
	MOVOU(C0, Mem{Base: oup}.Offset(2*16))
	MOVOU(Mem{Base: inp}.Offset(3*16), D3)
	PXOR(D3, D0)
	MOVOU(D0, Mem{Base: oup}.Offset(3*16))
	MOVOU(Mem{Base: inp}.Offset(4*16), D0)
	PXOR(D0, A1)
	MOVOU(A1, Mem{Base: oup}.Offset(4*16))
	MOVOU(Mem{Base: inp}.Offset(5*16), D0)
	PXOR(D0, B1)
	MOVOU(B1, Mem{Base: oup}.Offset(5*16))
	MOVOU(Mem{Base: inp}.Offset(6*16), D0)
	PXOR(D0, C1)
	MOVOU(C1, Mem{Base: oup}.Offset(6*16))
	MOVOU(Mem{Base: inp}.Offset(7*16), D0)
	PXOR(D0, D1)
	MOVOU(D1, Mem{Base: oup}.Offset(7*16))
	MOVOU(Mem{Base: inp}.Offset(8*16), D0)
	PXOR(D0, A2)
	MOVOU(A2, Mem{Base: oup}.Offset(8*16))
	MOVOU(Mem{Base: inp}.Offset(9*16), D0)
	PXOR(D0, B2)
	MOVOU(B2, Mem{Base: oup}.Offset(9*16))
	MOVOU(Mem{Base: inp}.Offset(10*16), D0)
	PXOR(D0, C2)
	MOVOU(C2, Mem{Base: oup}.Offset(10*16))
	MOVOU(Mem{Base: inp}.Offset(11*16), D0)
	PXOR(D0, D2)
	MOVOU(D2, Mem{Base: oup}.Offset(11*16))
	MOVOU(Mem{Base: inp}.Offset(12*16), D0)
	PXOR(D0, A3)
	MOVOU(A3, Mem{Base: oup}.Offset(12*16))
	MOVOU(Mem{Base: inp}.Offset(13*16), D0)
	PXOR(D0, B3)
	MOVOU(B3, Mem{Base: oup}.Offset(13*16))
	MOVOU(Mem{Base: inp}.Offset(14*16), D0)
	PXOR(D0, C3)
	MOVOU(C3, Mem{Base: oup}.Offset(14*16))
	MOVOU(Mem{Base: inp}.Offset(15*16), D0)
	PXOR(tmpStore, D0)
	MOVOU(D0, Mem{Base: oup}.Offset(15*16))
	LEAQ(Mem{Base: inp}.Offset(256), inp)
	LEAQ(Mem{Base: oup}.Offset(256), oup)
	SUBQ(U32(256), inl)
	JMP(LabelRef("openSSEMainLoop"))
}

func openSSEMainLoopDone() {
	Label("openSSEMainLoopDone")
	Comment("Handle the various tail sizes efficiently")
	TESTQ(inl, inl)
	JE(LabelRef("openSSEFinalize"))
	CMPQ(inl, Imm(64))
	JBE(LabelRef("openSSETail64"))
	CMPQ(inl, Imm(128))
	JBE(LabelRef("openSSETail128"))
	CMPQ(inl, Imm(192))
	JBE(LabelRef("openSSETail192"))
	JMP(LabelRef("openSSETail256"))
}

func openSSEFinalize() {
	Label("openSSEFinalize")
	Comment("Hash in the PT, AAD lengths")
	ADDQ(NewParamAddr("ad_len", 80), acc0)
	ADCQ(NewParamAddr("src_len", 56), acc1)
	ADCQ(Imm(1), acc2)
	polyMul()

	Comment("Final reduce")
	MOVQ(acc0, t0)
	MOVQ(acc1, t1)
	MOVQ(acc2, t2)
	SUBQ(I8(-5), acc0)
	SBBQ(I8(-1), acc1)
	SBBQ(Imm(3), acc2)
	CMOVQCS(t0, acc0)
	CMOVQCS(t1, acc1)
	CMOVQCS(t2, acc2)

	Comment("Add in the \"s\" part of the key")
	ADDQ(sStore.Offset(0), acc0)
	ADCQ(sStore.Offset(8), acc1)

	Comment("Finally, constant time compare to the tag at the end of the message")
	XORQ(RAX, RAX)
	MOVQ(U32(1), RDX)
	XORQ(Mem{Base: inp}.Offset(0*8), acc0)
	XORQ(Mem{Base: inp}.Offset(1*8), acc1)
	ORQ(acc1, acc0)
	CMOVQEQ(RDX, RAX)

	Comment("Return true iff tags are equal")
	// Hack to get Avo to emit:
	// 	MOVB AX, ret+96(FP)
	Instruction(&ir.Instruction{Opcode: "MOVB", Operands: []Op{AX, NewParamAddr("ret", 96)}})
	RET()
}

// ----------------------------------------------------------------------------
// Special optimization for buffers smaller than 129 bytes

// For up to 128 bytes of ciphertext and 64 bytes for the poly key, we require to process three blocks
func openSSE128() {
	Label("openSSE128")

	chacha20Constants := chacha20Constants_DATA()
	sseIncMask := sseIncMask_DATA()

	MOVOU(chacha20Constants, A0)
	MOVOU(Mem{Base: keyp}.Offset(1*16), B0)
	MOVOU(Mem{Base: keyp}.Offset(2*16), C0)
	MOVOU(Mem{Base: keyp}.Offset(3*16), D0)
	MOVO(A0, A1)
	MOVO(B0, B1)
	MOVO(C0, C1)
	MOVO(D0, D1)
	PADDL(sseIncMask, D1)
	MOVO(A1, A2)
	MOVO(B1, B2)
	MOVO(C1, C2)
	MOVO(D1, D2)
	PADDL(sseIncMask, D2)
	MOVO(B0, T1)
	MOVO(C0, T2)
	MOVO(D1, T3)
	MOVQ(U32(10), itr2)
}

func openSSE128InnerCipherLoop() {
	Label("openSSE128InnerCipherLoop")
	chachaQR(A0, B0, C0, D0, T0)
	chachaQR(A1, B1, C1, D1, T0)
	chachaQR(A2, B2, C2, D2, T0)
	shiftB0Left()
	shiftB1Left()
	shiftB2Left()
	shiftC0Left()
	shiftC1Left()
	shiftC2Left()
	shiftD0Left()
	shiftD1Left()
	shiftD2Left()
	chachaQR(A0, B0, C0, D0, T0)
	chachaQR(A1, B1, C1, D1, T0)
	chachaQR(A2, B2, C2, D2, T0)
	shiftB0Right()
	shiftB1Right()
	shiftB2Right()
	shiftC0Right()
	shiftC1Right()
	shiftC2Right()
	shiftD0Right()
	shiftD1Right()
	shiftD2Right()
	DECQ(itr2)
	JNE(LabelRef("openSSE128InnerCipherLoop"))

	Comment("A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded")

	chacha20Constants := chacha20Constants_DATA()
	PADDL(chacha20Constants, A0)
	PADDL(chacha20Constants, A1)
	PADDL(chacha20Constants, A2)
	PADDL(T1, B0)
	PADDL(T1, B1)
	PADDL(T1, B2)
	PADDL(T2, C1)
	PADDL(T2, C2)
	PADDL(T3, D1)
	sseIncMask := sseIncMask_DATA()
	PADDL(sseIncMask, T3)
	PADDL(T3, D2)

	Comment("Clamp and store the key")
	polyClampMask := polyClampMask_DATA()
	PAND(polyClampMask, A0)
	MOVOU(A0, rStore)
	MOVOU(B0, sStore)

	Comment("Hash")
	Load(Param("ad").Len(), itr2)
	CALL(LabelRef("polyHashADInternal<>(SB)"))
}

func openSSE128Open() {
	Label("openSSE128Open")
	CMPQ(inl, Imm(16))
	JB(LabelRef("openSSETail16"))
	SUBQ(Imm(16), inl)

	Comment("Load for hashing")
	polyAdd(Mem{Base: inp}.Offset(0))

	Comment("Load for decryption")
	MOVOU(Mem{Base: inp}, T0)
	PXOR(T0, A1)
	MOVOU(A1, Mem{Base: oup})
	LEAQ(Mem{Base: inp}.Offset(1*16), inp)
	LEAQ(Mem{Base: oup}.Offset(1*16), oup)
	polyMul()

	Comment("Shift the stream \"left\"")
	MOVO(B1, A1)
	MOVO(C1, B1)
	MOVO(D1, C1)
	MOVO(A2, D1)
	MOVO(B2, A2)
	MOVO(C2, B2)
	MOVO(D2, C2)
	JMP(LabelRef("openSSE128Open"))
}

func openSSETail16() {
	Label("openSSETail16")
	TESTQ(inl, inl)
	JE(LabelRef("openSSEFinalize"))

	Comment("We can safely load the CT from the end, because it is padded with the MAC")
	MOVQ(inl, itr2)
	SHLQ(Imm(4), itr2)
	andMask := andMask_DATA()
	LEAQ(andMask, t0)
	MOVOU(Mem{Base: inp}, T0)
	ADDQ(inl, inp)
	PAND(Mem{Base: t0, Index: itr2, Scale: 1}.Offset(-16), T0)
	MOVO(T0, tmpStore.Offset(0))
	MOVQ(T0, t0)
	MOVQ(tmpStore.Offset(8), t1)
	PXOR(A1, T0)
}

func openSSETail16Store() {
	Comment("We can only store one byte at a time, since plaintext can be shorter than 16 bytes")
	Label("openSSETail16Store")
	MOVQ(T0, t3)
	// Hack to get Avo to emit:
	// 	MOVB t3, (oup)
	Instruction(&ir.Instruction{Opcode: "MOVB", Operands: []Op{t3, Mem{Base: oup}}})
	PSRLDQ(Imm(1), T0)
	INCQ(oup)
	DECQ(inl)
	JNE(LabelRef("openSSETail16Store"))
	ADDQ(t0, acc0)
	ADCQ(t1, acc1)
	ADCQ(Imm(1), acc2)
	polyMul()
	JMP(LabelRef("openSSEFinalize"))
}

// ----------------------------------------------------------------------------
// Special optimization for the last 64 bytes of ciphertext

// Need to decrypt up to 64 bytes - prepare single block
func openSSETail64() {
	Label("openSSETail64")
	chacha20Constants := chacha20Constants_DATA()
	MOVO(chacha20Constants, A0)
	MOVO(state1Store, B0)
	MOVO(state2Store, C0)
	MOVO(ctr3Store, D0)
	sseIncMask := sseIncMask_DATA()
	PADDL(sseIncMask, D0)
	MOVO(D0, ctr0Store)
	XORQ(itr2, itr2)
	MOVQ(inl, itr1)
	CMPQ(itr1, Imm(16))
	JB(LabelRef("openSSETail64LoopB"))
}

// Perform ChaCha rounds, while hashing the remaining input
func openSSETail64LoopA() {
	Label("openSSETail64LoopA")
	polyAdd(Mem{Base: inp, Index: itr2, Scale: 1}.Offset(0))
	polyMul()
	SUBQ(Imm(16), itr1)
}

func openSSETail64LoopB() {
	Label("openSSETail64LoopB")
	ADDQ(Imm(16), itr2)
	chachaQR(A0, B0, C0, D0, T0)
	shiftB0Left()
	shiftC0Left()
	shiftD0Left()
	chachaQR(A0, B0, C0, D0, T0)
	shiftB0Right()
	shiftC0Right()
	shiftD0Right()

	CMPQ(itr1, Imm(16))
	JAE(LabelRef("openSSETail64LoopA"))

	CMPQ(itr2, Imm(160))
	JNE(LabelRef("openSSETail64LoopB"))

	chacha20Constants := chacha20Constants_DATA()
	PADDL(chacha20Constants, A0)
	PADDL(state1Store, B0)
	PADDL(state2Store, C0)
	PADDL(ctr0Store, D0)
}

func openSSETail64DecLoop() {
	Label("openSSETail64DecLoop")
	CMPQ(inl, Imm(16))
	JB(LabelRef("openSSETail64DecLoopDone"))
	SUBQ(Imm(16), inl)
	MOVOU(Mem{Base: inp}, T0)
	PXOR(T0, A0)
	MOVOU(A0, Mem{Base: oup})
	LEAQ(Mem{Base: inp}.Offset(16), inp)
	LEAQ(Mem{Base: oup}.Offset(16), oup)
	MOVO(B0, A0)
	MOVO(C0, B0)
	MOVO(D0, C0)
	JMP(LabelRef("openSSETail64DecLoop"))
}

func openSSETail64DecLoopDone() {
	Label("openSSETail64DecLoopDone")
	MOVO(A0, A1)
	JMP(LabelRef("openSSETail16"))
}

// ----------------------------------------------------------------------------
// Special optimization for the last 128 bytes of ciphertext

// Need to decrypt up to 128 bytes - prepare two blocks
func openSSETail128() {
	Label("openSSETail128")
	chacha20Constants := chacha20Constants_DATA()
	MOVO(chacha20Constants, A1)
	MOVO(state1Store, B1)
	MOVO(state2Store, C1)
	MOVO(ctr3Store, D1)
	sseIncMask := sseIncMask_DATA()
	PADDL(sseIncMask, D1)
	MOVO(D1, ctr0Store)
	MOVO(A1, A0)
	MOVO(B1, B0)
	MOVO(C1, C0)
	MOVO(D1, D0)
	PADDL(sseIncMask, D0)
	MOVO(D0, ctr1Store)
	XORQ(itr2, itr2)
	MOVQ(inl, itr1)
	ANDQ(I8(-16), itr1)
}

// Perform ChaCha rounds, while hashing the remaining input
func openSSETail128LoopA() {
	Label("openSSETail128LoopA")
	polyAdd(Mem{Base: inp, Index: itr2, Scale: 1}.Offset(0))
	polyMul()
}

func openSSETail128LoopB() {
	Label("openSSETail128LoopB")
	ADDQ(Imm(16), itr2)
	chachaQR(A0, B0, C0, D0, T0)
	chachaQR(A1, B1, C1, D1, T0)
	shiftB0Left()
	shiftC0Left()
	shiftD0Left()
	shiftB1Left()
	shiftC1Left()
	shiftD1Left()
	chachaQR(A0, B0, C0, D0, T0)
	chachaQR(A1, B1, C1, D1, T0)
	shiftB0Right()
	shiftC0Right()
	shiftD0Right()
	shiftB1Right()
	shiftC1Right()
	shiftD1Right()

	CMPQ(itr2, itr1)
	JB(LabelRef("openSSETail128LoopA"))

	CMPQ(itr2, Imm(160))
	JNE(LabelRef("openSSETail128LoopB"))

	chacha20Constants := chacha20Constants_DATA()
	PADDL(chacha20Constants, A0)
	PADDL(chacha20Constants, A1)
	PADDL(state1Store, B0)
	PADDL(state1Store, B1)
	PADDL(state2Store, C0)
	PADDL(state2Store, C1)
	PADDL(ctr1Store, D0)
	PADDL(ctr0Store, D1)

	MOVOU(Mem{Base: inp}.Offset(0*16), T0)
	MOVOU(Mem{Base: inp}.Offset(1*16), T1)
	MOVOU(Mem{Base: inp}.Offset(2*16), T2)
	MOVOU(Mem{Base: inp}.Offset(3*16), T3)
	PXOR(T0, A1)
	PXOR(T1, B1)
	PXOR(T2, C1)
	PXOR(T3, D1)
	MOVOU(A1, Mem{Base: oup}.Offset(0*16))
	MOVOU(B1, Mem{Base: oup}.Offset(1*16))
	MOVOU(C1, Mem{Base: oup}.Offset(2*16))
	MOVOU(D1, Mem{Base: oup}.Offset(3*16))

	SUBQ(Imm(64), inl)
	LEAQ(Mem{Base: inp}.Offset(64), inp)
	LEAQ(Mem{Base: oup}.Offset(64), oup)
	JMP(LabelRef("openSSETail64DecLoop"))
}

// ----------------------------------------------------------------------------
// Special optimization for the last 192 bytes of ciphertext

// Need to decrypt up to 192 bytes - prepare three blocks
func openSSETail192() {
	Label("openSSETail192")
	chacha20Constants := chacha20Constants_DATA()
	MOVO(chacha20Constants, A2)
	MOVO(state1Store, B2)
	MOVO(state2Store, C2)
	MOVO(ctr3Store, D2)
	sseIncMask := sseIncMask_DATA()
	PADDL(sseIncMask, D2)
	MOVO(D2, ctr0Store)
	MOVO(A2, A1)
	MOVO(B2, B1)
	MOVO(C2, C1)
	MOVO(D2, D1)
	PADDL(sseIncMask, D1)
	MOVO(D1, ctr1Store)
	MOVO(A1, A0)
	MOVO(B1, B0)
	MOVO(C1, C0)
	MOVO(D1, D0)
	PADDL(sseIncMask, D0)
	MOVO(D0, ctr2Store)

	MOVQ(inl, itr1)
	MOVQ(U32(160), itr2)
	CMPQ(itr1, Imm(160))
	CMOVQGT(itr2, itr1)
	ANDQ(I8(-16), itr1)
	XORQ(itr2, itr2)
}

// Perform ChaCha rounds, while hashing the remaining input
func openSSLTail192LoopA() {
	Label("openSSLTail192LoopA")
	polyAdd(Mem{Base: inp, Index: itr2, Scale: 1}.Offset(0))
	polyMul()
}

func openSSLTail192LoopB() {
	Label("openSSLTail192LoopB")
	ADDQ(Imm(16), itr2)
	chachaQR(A0, B0, C0, D0, T0)
	chachaQR(A1, B1, C1, D1, T0)
	chachaQR(A2, B2, C2, D2, T0)
	shiftB0Left()
	shiftC0Left()
	shiftD0Left()
	shiftB1Left()
	shiftC1Left()
	shiftD1Left()
	shiftB2Left()
	shiftC2Left()
	shiftD2Left()

	chachaQR(A0, B0, C0, D0, T0)
	chachaQR(A1, B1, C1, D1, T0)
	chachaQR(A2, B2, C2, D2, T0)
	shiftB0Right()
	shiftC0Right()
	shiftD0Right()
	shiftB1Right()
	shiftC1Right()
	shiftD1Right()
	shiftB2Right()
	shiftC2Right()
	shiftD2Right()

	CMPQ(itr2, itr1)
	JB(LabelRef("openSSLTail192LoopA"))

	CMPQ(itr2, Imm(160))
	JNE(LabelRef("openSSLTail192LoopB"))

	CMPQ(inl, Imm(176))
	JB(LabelRef("openSSLTail192Store"))

	polyAdd(Mem{Base: inp}.Offset(160))
	polyMul()

	CMPQ(inl, Imm(192))
	JB(LabelRef("openSSLTail192Store"))

	polyAdd(Mem{Base: inp}.Offset(176))
	polyMul()
}

func openSSLTail192Store() {
	Label("openSSLTail192Store")
	chacha20Constants := chacha20Constants_DATA()
	PADDL(chacha20Constants, A0)
	PADDL(chacha20Constants, A1)
	PADDL(chacha20Constants, A2)
	PADDL(state1Store, B0)
	PADDL(state1Store, B1)
	PADDL(state1Store, B2)
	PADDL(state2Store, C0)
	PADDL(state2Store, C1)
	PADDL(state2Store, C2)
	PADDL(ctr2Store, D0)
	PADDL(ctr1Store, D1)
	PADDL(ctr0Store, D2)

	MOVOU(Mem{Base: inp}.Offset(0*16), T0)
	MOVOU(Mem{Base: inp}.Offset(1*16), T1)
	MOVOU(Mem{Base: inp}.Offset(2*16), T2)
	MOVOU(Mem{Base: inp}.Offset(3*16), T3)
	PXOR(T0, A2)
	PXOR(T1, B2)
	PXOR(T2, C2)
	PXOR(T3, D2)
	MOVOU(A2, Mem{Base: oup}.Offset(0*16))
	MOVOU(B2, Mem{Base: oup}.Offset(1*16))
	MOVOU(C2, Mem{Base: oup}.Offset(2*16))
	MOVOU(D2, Mem{Base: oup}.Offset(3*16))

	MOVOU(Mem{Base: inp}.Offset(4*16), T0)
	MOVOU(Mem{Base: inp}.Offset(5*16), T1)
	MOVOU(Mem{Base: inp}.Offset(6*16), T2)
	MOVOU(Mem{Base: inp}.Offset(7*16), T3)
	PXOR(T0, A1)
	PXOR(T1, B1)
	PXOR(T2, C1)
	PXOR(T3, D1)
	MOVOU(A1, Mem{Base: oup}.Offset(4*16))
	MOVOU(B1, Mem{Base: oup}.Offset(5*16))
	MOVOU(C1, Mem{Base: oup}.Offset(6*16))
	MOVOU(D1, Mem{Base: oup}.Offset(7*16))

	SUBQ(Imm(128), inl)
	LEAQ(Mem{Base: inp}.Offset(128), inp)
	LEAQ(Mem{Base: oup}.Offset(128), oup)
	JMP(LabelRef("openSSETail64DecLoop"))
}

// ----------------------------------------------------------------------------
// Special optimization for the last 256 bytes of ciphertext

// Need to decrypt up to 256 bytes - prepare four blocks
func openSSETail256() {
	Label("openSSETail256")
	chacha20Constants := chacha20Constants_DATA()
	MOVO(chacha20Constants, A0)
	MOVO(state1Store, B0)
	MOVO(state2Store, C0)
	MOVO(ctr3Store, D0)
	sseIncMask := sseIncMask_DATA()
	PADDL(sseIncMask, D0)
	MOVO(A0, A1)
	MOVO(B0, B1)
	MOVO(C0, C1)
	MOVO(D0, D1)
	PADDL(sseIncMask, D1)
	MOVO(A1, A2)
	MOVO(B1, B2)
	MOVO(C1, C2)
	MOVO(D1, D2)
	PADDL(sseIncMask, D2)
	MOVO(A2, A3)
	MOVO(B2, B3)
	MOVO(C2, C3)
	MOVO(D2, D3)
	PADDL(sseIncMask, D3)

	Comment("Store counters")
	MOVO(D0, ctr0Store)
	MOVO(D1, ctr1Store)
	MOVO(D2, ctr2Store)
	MOVO(D3, ctr3Store)
	XORQ(itr2, itr2)
}

// This loop inteleaves 8 ChaCha quarter rounds with 1 poly multiplication
func openSSETail256Loop() {
	Label("openSSETail256Loop")
	polyAdd(Mem{Base: inp, Index: itr2, Scale: 1}.Offset(0))
	MOVO(C3, tmpStore)
	chachaQR(A0, B0, C0, D0, C3)
	chachaQR(A1, B1, C1, D1, C3)
	chachaQR(A2, B2, C2, D2, C3)
	MOVO(tmpStore, C3)
	MOVO(C1, tmpStore)
	chachaQR(A3, B3, C3, D3, C1)
	MOVO(tmpStore, C1)
	shiftB0Left()
	shiftB1Left()
	shiftB2Left()
	shiftB3Left()
	shiftC0Left()
	shiftC1Left()
	shiftC2Left()
	shiftC3Left()
	shiftD0Left()
	shiftD1Left()
	shiftD2Left()
	shiftD3Left()
	polyMulStage1()
	polyMulStage2()
	MOVO(C3, tmpStore)
	chachaQR(A0, B0, C0, D0, C3)
	chachaQR(A1, B1, C1, D1, C3)
	chachaQR(A2, B2, C2, D2, C3)
	MOVO(tmpStore, C3)
	MOVO(C1, tmpStore)
	chachaQR(A3, B3, C3, D3, C1)
	MOVO(tmpStore, C1)
	polyMulStage3()
	polyMulReduceStage()
	shiftB0Right()
	shiftB1Right()
	shiftB2Right()
	shiftB3Right()
	shiftC0Right()
	shiftC1Right()
	shiftC2Right()
	shiftC3Right()
	shiftD0Right()
	shiftD1Right()
	shiftD2Right()
	shiftD3Right()
	ADDQ(Imm(2*8), itr2)
	CMPQ(itr2, Imm(160))
	JB(LabelRef("openSSETail256Loop"))
	MOVQ(inl, itr1)
	ANDQ(I8(-16), itr1)
}

func openSSETail256HashLoop() {
	Label("openSSETail256HashLoop")
	polyAdd(Mem{Base: inp, Index: itr2, Scale: 1}.Offset(0))
	polyMul()
	ADDQ(Imm(2*8), itr2)
	CMPQ(itr2, itr1)
	JB(LabelRef("openSSETail256HashLoop"))

	Comment("Add in the state")
	chacha20Constants := chacha20Constants_DATA()
	PADDD(chacha20Constants, A0)
	PADDD(chacha20Constants, A1)
	PADDD(chacha20Constants, A2)
	PADDD(chacha20Constants, A3)
	PADDD(state1Store, B0)
	PADDD(state1Store, B1)
	PADDD(state1Store, B2)
	PADDD(state1Store, B3)
	PADDD(state2Store, C0)
	PADDD(state2Store, C1)
	PADDD(state2Store, C2)
	PADDD(state2Store, C3)
	PADDD(ctr0Store, D0)
	PADDD(ctr1Store, D1)
	PADDD(ctr2Store, D2)
	PADDD(ctr3Store, D3)
	MOVO(D3, tmpStore)

	Comment("Load - xor - store")
	MOVOU(Mem{Base: inp}.Offset(0*16), D3)
	PXOR(D3, A0)
	MOVOU(Mem{Base: inp}.Offset(1*16), D3)
	PXOR(D3, B0)
	MOVOU(Mem{Base: inp}.Offset(2*16), D3)
	PXOR(D3, C0)
	MOVOU(Mem{Base: inp}.Offset(3*16), D3)
	PXOR(D3, D0)
	MOVOU(A0, Mem{Base: oup}.Offset(0*16))
	MOVOU(B0, Mem{Base: oup}.Offset(1*16))
	MOVOU(C0, Mem{Base: oup}.Offset(2*16))
	MOVOU(D0, Mem{Base: oup}.Offset(3*16))
	MOVOU(Mem{Base: inp}.Offset(4*16), A0)
	MOVOU(Mem{Base: inp}.Offset(5*16), B0)
	MOVOU(Mem{Base: inp}.Offset(6*16), C0)
	MOVOU(Mem{Base: inp}.Offset(7*16), D0)
	PXOR(A0, A1)
	PXOR(B0, B1)
	PXOR(C0, C1)
	PXOR(D0, D1)
	MOVOU(A1, Mem{Base: oup}.Offset(4*16))
	MOVOU(B1, Mem{Base: oup}.Offset(5*16))
	MOVOU(C1, Mem{Base: oup}.Offset(6*16))
	MOVOU(D1, Mem{Base: oup}.Offset(7*16))
	MOVOU(Mem{Base: inp}.Offset(8*16), A0)
	MOVOU(Mem{Base: inp}.Offset(9*16), B0)
	MOVOU(Mem{Base: inp}.Offset(10*16), C0)
	MOVOU(Mem{Base: inp}.Offset(11*16), D0)
	PXOR(A0, A2)
	PXOR(B0, B2)
	PXOR(C0, C2)
	PXOR(D0, D2)
	MOVOU(A2, Mem{Base: oup}.Offset(8*16))
	MOVOU(B2, Mem{Base: oup}.Offset(9*16))
	MOVOU(C2, Mem{Base: oup}.Offset(10*16))
	MOVOU(D2, Mem{Base: oup}.Offset(11*16))
	LEAQ(Mem{Base: inp}.Offset(192), inp)
	LEAQ(Mem{Base: oup}.Offset(192), oup)
	SUBQ(Imm(192), inl)
	MOVO(A3, A0)
	MOVO(B3, B0)
	MOVO(C3, C0)
	MOVO(tmpStore, D0)

	JMP(LabelRef("openSSETail64DecLoop"))
}

// Functions to emit AVX instructions via BYTE directive

// broadcasti128 16(r8), ymm14
func VBROADCASTI128_16_R8_YMM14() {
	BYTE(U8(0xc4))
	BYTE(U8(0x42))
	BYTE(U8(0x7d))
	BYTE(U8(0x5a))
	BYTE(U8(0x70))
	BYTE(U8(0x10))
}

// broadcasti128 32(r8), ymm12
func VBROADCASTI128_32_R8_YMM12() {
	BYTE(U8(0xc4))
	BYTE(U8(0x42))
	BYTE(U8(0x7d))
	BYTE(U8(0x5a))
	BYTE(U8(0x60))
	BYTE(U8(0x20))
}

// broadcasti128 48(r8), ymm4
func VBROADCASTI128_48_R8_YMM4() {
	BYTE(U8(0xc4))
	BYTE(U8(0xc2))
	BYTE(U8(0x7d))
	BYTE(U8(0x5a))
	BYTE(U8(0x60))
	BYTE(U8(0x30))
}

// ----------------------------------------------------------------------------
// ------------------------- AVX2 Code ----------------------------------------

func chacha20Poly1305Open_AVX2() {
	Label("chacha20Poly1305Open_AVX2")
	VZEROUPPER()
	chacha20Constants := chacha20Constants_DATA()
	VMOVDQU(chacha20Constants, AA0)
	VBROADCASTI128_16_R8_YMM14()
	VBROADCASTI128_32_R8_YMM12()
	VBROADCASTI128_48_R8_YMM4()
	avx2InitMask := avx2InitMask_DATA()
	VPADDD(avx2InitMask, DD0, DD0)

	Comment("Special optimization, for very short buffers")
	CMPQ(inl, Imm(192))
	JBE(LabelRef("openAVX2192"))
	CMPQ(inl, U32(320))
	JBE(LabelRef("openAVX2320"))

	Comment("For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream")
	VMOVDQA(BB0, state1StoreAVX2)
	VMOVDQA(CC0, state2StoreAVX2)
	VMOVDQA(DD0, ctr3StoreAVX2)
	MOVQ(U32(10), itr2)
}

func openAVX2PreparePolyKey() {
	Label("openAVX2PreparePolyKey")
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	VPALIGNR(Imm(4), BB0, BB0, BB0)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(12), DD0, DD0, DD0)
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	VPALIGNR(Imm(12), BB0, BB0, BB0)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(4), DD0, DD0, DD0)
	DECQ(itr2)
	JNE(LabelRef("openAVX2PreparePolyKey"))

	chacha20Constants := chacha20Constants_DATA()
	VPADDD(chacha20Constants, AA0, AA0)
	VPADDD(state1StoreAVX2, BB0, BB0)
	VPADDD(state2StoreAVX2, CC0, CC0)
	VPADDD(ctr3StoreAVX2, DD0, DD0)

	VPERM2I128(Imm(0x02), AA0, BB0, TT0)

	Comment("Clamp and store poly key")
	polyClampMask := polyClampMask_DATA()
	VPAND(polyClampMask, TT0, TT0)
	VMOVDQA(TT0, rsStoreAVX2)

	Comment("Stream for the first 64 bytes")
	VPERM2I128(Imm(0x13), AA0, BB0, AA0)
	VPERM2I128(Imm(0x13), CC0, DD0, BB0)

	Comment("Hash AD + first 64 bytes")
	// MOVQ ad_len+80(FP), itr2
	MOVQ(NewParamAddr("ad_len", 80), itr2)
	CALL(LabelRef("polyHashADInternal<>(SB)"))
	XORQ(itr1, itr1)
}

func openAVX2InitialHash64() {
	Label("openAVX2InitialHash64")
	// polyAdd(0(inp)(itr1*1))
	polyAdd(Mem{Base: inp, Index: itr1, Scale: 1}.Offset(0))
	polyMulAVX2()
	ADDQ(Imm(16), itr1)
	CMPQ(itr1, Imm(64))
	JNE(LabelRef("openAVX2InitialHash64"))

	Comment("Decrypt the first 64 bytes")
	VPXOR(Mem{Base: inp}.Offset(0*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(1*32), BB0, BB0)
	VMOVDQU(AA0, Mem{Base: oup}.Offset(0*32))
	VMOVDQU(BB0, Mem{Base: oup}.Offset(1*32))
	LEAQ(Mem{Base: inp}.Offset(2*32), inp)
	LEAQ(Mem{Base: oup}.Offset(2*32), oup)
	SUBQ(Imm(64), inl)
}

func openAVX2MainLoop() {
	Label("openAVX2MainLoop")
	CMPQ(inl, U32(512))
	JB(LabelRef("openAVX2MainLoopDone"))

	Comment("Load state, increment counter blocks, store the incremented counters")
	chacha20Constants := chacha20Constants_DATA()
	VMOVDQU(chacha20Constants, AA0)
	VMOVDQA(AA0, AA1)
	VMOVDQA(AA0, AA2)
	VMOVDQA(AA0, AA3)
	VMOVDQA(state1StoreAVX2, BB0)
	VMOVDQA(BB0, BB1)
	VMOVDQA(BB0, BB2)
	VMOVDQA(BB0, BB3)
	VMOVDQA(state2StoreAVX2, CC0)
	VMOVDQA(CC0, CC1)
	VMOVDQA(CC0, CC2)
	VMOVDQA(CC0, CC3)
	VMOVDQA(ctr3StoreAVX2, DD0)
	avx2IncMask := avx2IncMask_DATA()
	VPADDD(avx2IncMask, DD0, DD0)
	VPADDD(avx2IncMask, DD0, DD1)
	VPADDD(avx2IncMask, DD1, DD2)
	VPADDD(avx2IncMask, DD2, DD3)
	VMOVDQA(DD0, ctr0StoreAVX2)
	VMOVDQA(DD1, ctr1StoreAVX2)
	VMOVDQA(DD2, ctr2StoreAVX2)
	VMOVDQA(DD3, ctr3StoreAVX2)
	XORQ(itr1, itr1)
}

// Lets just say this spaghetti loop interleaves 2 quarter rounds with 3 poly multiplications
// Effectively per 512 bytes of stream we hash 480 bytes of ciphertext
func openAVX2InternalLoop() {
	Label("openAVX2InternalLoop")
	polyAdd(Mem{Base: inp, Index: itr1, Scale: 1}.Offset(0 * 8))
	VPADDD(BB0, AA0, AA0)
	VPADDD(BB1, AA1, AA1)
	VPADDD(BB2, AA2, AA2)
	VPADDD(BB3, AA3, AA3)
	polyMulStage1_AVX2()
	VPXOR(AA0, DD0, DD0)
	VPXOR(AA1, DD1, DD1)
	VPXOR(AA2, DD2, DD2)
	VPXOR(AA3, DD3, DD3)
	rol16 := rol16_DATA()
	VPSHUFB(rol16, DD0, DD0)
	VPSHUFB(rol16, DD1, DD1)
	VPSHUFB(rol16, DD2, DD2)
	VPSHUFB(rol16, DD3, DD3)
	polyMulStage2_AVX2()
	VPADDD(DD0, CC0, CC0)
	VPADDD(DD1, CC1, CC1)
	VPADDD(DD2, CC2, CC2)
	VPADDD(DD3, CC3, CC3)
	VPXOR(CC0, BB0, BB0)
	VPXOR(CC1, BB1, BB1)
	VPXOR(CC2, BB2, BB2)
	VPXOR(CC3, BB3, BB3)
	polyMulStage3_AVX2()
	VMOVDQA(CC3, tmpStoreAVX2)
	VPSLLD(Imm(12), BB0, CC3)
	VPSRLD(Imm(20), BB0, BB0)
	VPXOR(CC3, BB0, BB0)
	VPSLLD(Imm(12), BB1, CC3)
	VPSRLD(Imm(20), BB1, BB1)
	VPXOR(CC3, BB1, BB1)
	VPSLLD(Imm(12), BB2, CC3)
	VPSRLD(Imm(20), BB2, BB2)
	VPXOR(CC3, BB2, BB2)
	VPSLLD(Imm(12), BB3, CC3)
	VPSRLD(Imm(20), BB3, BB3)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(tmpStoreAVX2, CC3)
	polyMulReduceStage()
	VPADDD(BB0, AA0, AA0)
	VPADDD(BB1, AA1, AA1)
	VPADDD(BB2, AA2, AA2)
	VPADDD(BB3, AA3, AA3)
	VPXOR(AA0, DD0, DD0)
	VPXOR(AA1, DD1, DD1)
	VPXOR(AA2, DD2, DD2)
	VPXOR(AA3, DD3, DD3)
	rol8 := rol8_DATA()
	VPSHUFB(rol8, DD0, DD0)
	VPSHUFB(rol8, DD1, DD1)
	VPSHUFB(rol8, DD2, DD2)
	VPSHUFB(rol8, DD3, DD3)
	polyAdd(Mem{Base: inp, Index: itr1, Scale: 1}.Offset(2 * 8))
	VPADDD(DD0, CC0, CC0)
	VPADDD(DD1, CC1, CC1)
	VPADDD(DD2, CC2, CC2)
	VPADDD(DD3, CC3, CC3)
	polyMulStage1_AVX2()
	VPXOR(CC0, BB0, BB0)
	VPXOR(CC1, BB1, BB1)
	VPXOR(CC2, BB2, BB2)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(CC3, tmpStoreAVX2)
	VPSLLD(Imm(7), BB0, CC3)
	VPSRLD(Imm(25), BB0, BB0)
	VPXOR(CC3, BB0, BB0)
	VPSLLD(Imm(7), BB1, CC3)
	VPSRLD(Imm(25), BB1, BB1)
	VPXOR(CC3, BB1, BB1)
	VPSLLD(Imm(7), BB2, CC3)
	VPSRLD(Imm(25), BB2, BB2)
	VPXOR(CC3, BB2, BB2)
	VPSLLD(Imm(7), BB3, CC3)
	VPSRLD(Imm(25), BB3, BB3)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(tmpStoreAVX2, CC3)
	polyMulStage2_AVX2()
	VPALIGNR(Imm(4), BB0, BB0, BB0)
	VPALIGNR(Imm(4), BB1, BB1, BB1)
	VPALIGNR(Imm(4), BB2, BB2, BB2)
	VPALIGNR(Imm(4), BB3, BB3, BB3)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(8), CC3, CC3, CC3)
	VPALIGNR(Imm(12), DD0, DD0, DD0)
	VPALIGNR(Imm(12), DD1, DD1, DD1)
	VPALIGNR(Imm(12), DD2, DD2, DD2)
	VPALIGNR(Imm(12), DD3, DD3, DD3)
	VPADDD(BB0, AA0, AA0)
	VPADDD(BB1, AA1, AA1)
	VPADDD(BB2, AA2, AA2)
	VPADDD(BB3, AA3, AA3)
	polyMulStage3_AVX2()
	VPXOR(AA0, DD0, DD0)
	VPXOR(AA1, DD1, DD1)
	VPXOR(AA2, DD2, DD2)
	VPXOR(AA3, DD3, DD3)
	VPSHUFB(rol16, DD0, DD0)
	VPSHUFB(rol16, DD1, DD1)
	VPSHUFB(rol16, DD2, DD2)
	VPSHUFB(rol16, DD3, DD3)
	polyMulReduceStage()
	VPADDD(DD0, CC0, CC0)
	VPADDD(DD1, CC1, CC1)
	VPADDD(DD2, CC2, CC2)
	VPADDD(DD3, CC3, CC3)
	VPXOR(CC0, BB0, BB0)
	VPXOR(CC1, BB1, BB1)
	VPXOR(CC2, BB2, BB2)
	VPXOR(CC3, BB3, BB3)
	polyAdd(Mem{Base: inp, Index: itr1, Scale: 1}.Offset(4 * 8))
	LEAQ(Mem{Base: itr1}.Offset(6*8), itr1)
	VMOVDQA(CC3, tmpStoreAVX2)
	VPSLLD(Imm(12), BB0, CC3)
	VPSRLD(Imm(20), BB0, BB0)
	VPXOR(CC3, BB0, BB0)
	VPSLLD(Imm(12), BB1, CC3)
	VPSRLD(Imm(20), BB1, BB1)
	VPXOR(CC3, BB1, BB1)
	VPSLLD(Imm(12), BB2, CC3)
	VPSRLD(Imm(20), BB2, BB2)
	VPXOR(CC3, BB2, BB2)
	VPSLLD(Imm(12), BB3, CC3)
	VPSRLD(Imm(20), BB3, BB3)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(tmpStoreAVX2, CC3)
	polyMulStage1_AVX2()
	VPADDD(BB0, AA0, AA0)
	VPADDD(BB1, AA1, AA1)
	VPADDD(BB2, AA2, AA2)
	VPADDD(BB3, AA3, AA3)
	VPXOR(AA0, DD0, DD0)
	VPXOR(AA1, DD1, DD1)
	VPXOR(AA2, DD2, DD2)
	VPXOR(AA3, DD3, DD3)
	polyMulStage2_AVX2()
	VPSHUFB(rol8, DD0, DD0)
	VPSHUFB(rol8, DD1, DD1)
	VPSHUFB(rol8, DD2, DD2)
	VPSHUFB(rol8, DD3, DD3)
	VPADDD(DD0, CC0, CC0)
	VPADDD(DD1, CC1, CC1)
	VPADDD(DD2, CC2, CC2)
	VPADDD(DD3, CC3, CC3)
	polyMulStage3_AVX2()
	VPXOR(CC0, BB0, BB0)
	VPXOR(CC1, BB1, BB1)
	VPXOR(CC2, BB2, BB2)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(CC3, tmpStoreAVX2)
	VPSLLD(Imm(7), BB0, CC3)
	VPSRLD(Imm(25), BB0, BB0)
	VPXOR(CC3, BB0, BB0)
	VPSLLD(Imm(7), BB1, CC3)
	VPSRLD(Imm(25), BB1, BB1)
	VPXOR(CC3, BB1, BB1)
	VPSLLD(Imm(7), BB2, CC3)
	VPSRLD(Imm(25), BB2, BB2)
	VPXOR(CC3, BB2, BB2)
	VPSLLD(Imm(7), BB3, CC3)
	VPSRLD(Imm(25), BB3, BB3)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(tmpStoreAVX2, CC3)
	polyMulReduceStage()
	VPALIGNR(Imm(12), BB0, BB0, BB0)
	VPALIGNR(Imm(12), BB1, BB1, BB1)
	VPALIGNR(Imm(12), BB2, BB2, BB2)
	VPALIGNR(Imm(12), BB3, BB3, BB3)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(8), CC3, CC3, CC3)
	VPALIGNR(Imm(4), DD0, DD0, DD0)
	VPALIGNR(Imm(4), DD1, DD1, DD1)
	VPALIGNR(Imm(4), DD2, DD2, DD2)
	VPALIGNR(Imm(4), DD3, DD3, DD3)
	CMPQ(itr1, U32(480))
	JNE(LabelRef("openAVX2InternalLoop"))

	chacha20Constants := chacha20Constants_DATA()
	VPADDD(chacha20Constants, AA0, AA0)
	VPADDD(chacha20Constants, AA1, AA1)
	VPADDD(chacha20Constants, AA2, AA2)
	VPADDD(chacha20Constants, AA3, AA3)
	VPADDD(state1StoreAVX2, BB0, BB0)
	VPADDD(state1StoreAVX2, BB1, BB1)
	VPADDD(state1StoreAVX2, BB2, BB2)
	VPADDD(state1StoreAVX2, BB3, BB3)
	VPADDD(state2StoreAVX2, CC0, CC0)
	VPADDD(state2StoreAVX2, CC1, CC1)
	VPADDD(state2StoreAVX2, CC2, CC2)
	VPADDD(state2StoreAVX2, CC3, CC3)
	VPADDD(ctr0StoreAVX2, DD0, DD0)
	VPADDD(ctr1StoreAVX2, DD1, DD1)
	VPADDD(ctr2StoreAVX2, DD2, DD2)
	VPADDD(ctr3StoreAVX2, DD3, DD3)
	VMOVDQA(CC3, tmpStoreAVX2)

	Comment("We only hashed 480 of the 512 bytes available - hash the remaining 32 here")
	polyAdd(Mem{Base: inp}.Offset(480))
	polyMulAVX2()
	VPERM2I128(Imm(0x02), AA0, BB0, CC3)
	VPERM2I128(Imm(0x13), AA0, BB0, BB0)
	VPERM2I128(Imm(0x02), CC0, DD0, AA0)
	VPERM2I128(Imm(0x13), CC0, DD0, CC0)
	VPXOR(Mem{Base: inp}.Offset(0*32), CC3, CC3)
	VPXOR(Mem{Base: inp}.Offset(1*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(2*32), BB0, BB0)
	VPXOR(Mem{Base: inp}.Offset(3*32), CC0, CC0)
	VMOVDQU(CC3, Mem{Base: oup}.Offset(0*32))
	VMOVDQU(AA0, Mem{Base: oup}.Offset(1*32))
	VMOVDQU(BB0, Mem{Base: oup}.Offset(2*32))
	VMOVDQU(CC0, Mem{Base: oup}.Offset(3*32))
	VPERM2I128(Imm(0x02), AA1, BB1, AA0)
	VPERM2I128(Imm(0x02), CC1, DD1, BB0)
	VPERM2I128(Imm(0x13), AA1, BB1, CC0)
	VPERM2I128(Imm(0x13), CC1, DD1, DD0)
	VPXOR(Mem{Base: inp}.Offset(4*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(5*32), BB0, BB0)
	VPXOR(Mem{Base: inp}.Offset(6*32), CC0, CC0)
	VPXOR(Mem{Base: inp}.Offset(7*32), DD0, DD0)
	VMOVDQU(AA0, Mem{Base: oup}.Offset(4*32))
	VMOVDQU(BB0, Mem{Base: oup}.Offset(5*32))
	VMOVDQU(CC0, Mem{Base: oup}.Offset(6*32))
	VMOVDQU(DD0, Mem{Base: oup}.Offset(7*32))

	Comment("and here")
	polyAdd(Mem{Base: inp}.Offset(496))
	polyMulAVX2()
	VPERM2I128(Imm(0x02), AA2, BB2, AA0)
	VPERM2I128(Imm(0x02), CC2, DD2, BB0)
	VPERM2I128(Imm(0x13), AA2, BB2, CC0)
	VPERM2I128(Imm(0x13), CC2, DD2, DD0)
	VPXOR(Mem{Base: inp}.Offset(8*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(9*32), BB0, BB0)
	VPXOR(Mem{Base: inp}.Offset(10*32), CC0, CC0)
	VPXOR(Mem{Base: inp}.Offset(11*32), DD0, DD0)
	VMOVDQU(AA0, Mem{Base: oup}.Offset(8*32))
	VMOVDQU(BB0, Mem{Base: oup}.Offset(9*32))
	VMOVDQU(CC0, Mem{Base: oup}.Offset(10*32))
	VMOVDQU(DD0, Mem{Base: oup}.Offset(11*32))
	VPERM2I128(Imm(0x02), AA3, BB3, AA0)
	VPERM2I128(Imm(0x02), tmpStoreAVX2, DD3, BB0)
	VPERM2I128(Imm(0x13), AA3, BB3, CC0)
	VPERM2I128(Imm(0x13), tmpStoreAVX2, DD3, DD0)
	VPXOR(Mem{Base: inp}.Offset(12*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(13*32), BB0, BB0)
	VPXOR(Mem{Base: inp}.Offset(14*32), CC0, CC0)
	VPXOR(Mem{Base: inp}.Offset(15*32), DD0, DD0)
	VMOVDQU(AA0, Mem{Base: oup}.Offset(12*32))
	VMOVDQU(BB0, Mem{Base: oup}.Offset(13*32))
	VMOVDQU(CC0, Mem{Base: oup}.Offset(14*32))
	VMOVDQU(DD0, Mem{Base: oup}.Offset(15*32))
	LEAQ(Mem{Base: inp}.Offset(32*16), inp)
	LEAQ(Mem{Base: oup}.Offset(32*16), oup)
	SUBQ(U32(32*16), inl)
	JMP(LabelRef("openAVX2MainLoop"))
}

// Handle the various tail sizes efficiently
func openAVX2MainLoopDone() {
	Label("openAVX2MainLoopDone")
	Comment("Handle the various tail sizes efficiently")
	TESTQ(inl, inl)
	JE(LabelRef("openSSEFinalize"))
	CMPQ(inl, Imm(128))
	JBE(LabelRef("openAVX2Tail128"))
	CMPQ(inl, U32(256))
	JBE(LabelRef("openAVX2Tail256"))
	CMPQ(inl, U32(384))
	JBE(LabelRef("openAVX2Tail384"))
	JMP(LabelRef("openAVX2Tail512"))
}

// ----------------------------------------------------------------------------
// Special optimization for buffers smaller than 193 bytes

// For up to 192 bytes of ciphertext and 64 bytes for the poly key, we process four blocks
func openAVX2192() {
	Label("openAVX2192")
	VMOVDQA(AA0, AA1)
	VMOVDQA(BB0, BB1)
	VMOVDQA(CC0, CC1)
	avx2IncMask := avx2IncMask_DATA()
	VPADDD(avx2IncMask, DD0, DD1)
	VMOVDQA(AA0, AA2)
	VMOVDQA(BB0, BB2)
	VMOVDQA(CC0, CC2)
	VMOVDQA(DD0, DD2)
	VMOVDQA(DD1, TT3)
	MOVQ(U32(10), itr2)
}

func openAVX2192InnerCipherLoop() {
	Label("openAVX2192InnerCipherLoop")
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	VPALIGNR(Imm(4), BB0, BB0, BB0)
	VPALIGNR(Imm(4), BB1, BB1, BB1)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(12), DD0, DD0, DD0)
	VPALIGNR(Imm(12), DD1, DD1, DD1)
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	VPALIGNR(Imm(12), BB0, BB0, BB0)
	VPALIGNR(Imm(12), BB1, BB1, BB1)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(4), DD0, DD0, DD0)
	VPALIGNR(Imm(4), DD1, DD1, DD1)
	DECQ(itr2)
	JNE(LabelRef("openAVX2192InnerCipherLoop"))
	VPADDD(AA2, AA0, AA0)
	VPADDD(AA2, AA1, AA1)
	VPADDD(BB2, BB0, BB0)
	VPADDD(BB2, BB1, BB1)
	VPADDD(CC2, CC0, CC0)
	VPADDD(CC2, CC1, CC1)
	VPADDD(DD2, DD0, DD0)
	VPADDD(TT3, DD1, DD1)
	VPERM2I128(Imm(0x02), AA0, BB0, TT0)

	Comment("Clamp and store poly key")
	polyClampMask := polyClampMask_DATA()
	VPAND(polyClampMask, TT0, TT0)
	VMOVDQA(TT0, rsStoreAVX2)

	Comment("Stream for up to 192 bytes")
	VPERM2I128(Imm(0x13), AA0, BB0, AA0)
	VPERM2I128(Imm(0x13), CC0, DD0, BB0)
	VPERM2I128(Imm(0x02), AA1, BB1, CC0)
	VPERM2I128(Imm(0x02), CC1, DD1, DD0)
	VPERM2I128(Imm(0x13), AA1, BB1, AA1)
	VPERM2I128(Imm(0x13), CC1, DD1, BB1)
}

func openAVX2ShortOpen() {
	Label("openAVX2ShortOpen")
	Comment("Hash")
	Load(Param("ad").Len(), itr2)
	CALL(LabelRef("polyHashADInternal<>(SB)"))
}

func openAVX2ShortOpenLoop() {
	Label("openAVX2ShortOpenLoop")
	CMPQ(inl, Imm(32))
	JB(LabelRef("openAVX2ShortTail32"))
	SUBQ(Imm(32), inl)

	Comment("Load for hashing")
	polyAdd(Mem{Base: inp}.Offset(0 * 8))
	polyMulAVX2()
	polyAdd(Mem{Base: inp}.Offset(2 * 8))
	polyMulAVX2()

	Comment("Load for decryption")
	VPXOR(Mem{Base: inp}, AA0, AA0)
	VMOVDQU(AA0, Mem{Base: oup})
	LEAQ(Mem{Base: inp}.Offset(1*32), inp)
	LEAQ(Mem{Base: oup}.Offset(1*32), oup)

	Comment("Shift stream left")
	VMOVDQA(BB0, AA0)
	VMOVDQA(CC0, BB0)
	VMOVDQA(DD0, CC0)
	VMOVDQA(AA1, DD0)
	VMOVDQA(BB1, AA1)
	VMOVDQA(CC1, BB1)
	VMOVDQA(DD1, CC1)
	VMOVDQA(AA2, DD1)
	VMOVDQA(BB2, AA2)
	JMP(LabelRef("openAVX2ShortOpenLoop"))
}

func openAVX2ShortTail32() {
	Label("openAVX2ShortTail32")
	CMPQ(inl, Imm(16))
	VMOVDQA(A0, A1)
	JB(LabelRef("openAVX2ShortDone"))

	SUBQ(Imm(16), inl)

	Comment("Load for hashing")
	polyAdd(Mem{Base: inp}.Offset(0 * 8))
	polyMulAVX2()

	Comment("Load for decryption")
	VPXOR(Mem{Base: inp}, A0, T0)
	VMOVDQU(T0, Mem{Base: oup})
	LEAQ(Mem{Base: inp}.Offset(1*16), inp)
	LEAQ(Mem{Base: oup}.Offset(1*16), oup)
	VPERM2I128(Imm(0x11), AA0, AA0, AA0)
	VMOVDQA(A0, A1)
}

func openAVX2ShortDone() {
	Label("openAVX2ShortDone")
	VZEROUPPER()
	JMP(LabelRef("openSSETail16"))
}

// ----------------------------------------------------------------------------
// Special optimization for buffers smaller than 321 bytes

// For up to 320 bytes of ciphertext and 64 bytes for the poly key, we process six blocks
func openAVX2320() {
	Label("openAVX2320")
	VMOVDQA(AA0, AA1)
	VMOVDQA(BB0, BB1)
	VMOVDQA(CC0, CC1)
	avx2IncMask := avx2IncMask_DATA()
	VPADDD(avx2IncMask, DD0, DD1)
	VMOVDQA(AA0, AA2)
	VMOVDQA(BB0, BB2)
	VMOVDQA(CC0, CC2)
	VPADDD(avx2IncMask, DD1, DD2)
	VMOVDQA(BB0, TT1)
	VMOVDQA(CC0, TT2)
	VMOVDQA(DD0, TT3)
	MOVQ(U32(10), itr2)
}

func openAVX2320InnerCipherLoop() {
	Label("openAVX2320InnerCipherLoop")
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
	VPALIGNR(Imm(4), BB0, BB0, BB0)
	VPALIGNR(Imm(4), BB1, BB1, BB1)
	VPALIGNR(Imm(4), BB2, BB2, BB2)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(12), DD0, DD0, DD0)
	VPALIGNR(Imm(12), DD1, DD1, DD1)
	VPALIGNR(Imm(12), DD2, DD2, DD2)
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
	VPALIGNR(Imm(12), BB0, BB0, BB0)
	VPALIGNR(Imm(12), BB1, BB1, BB1)
	VPALIGNR(Imm(12), BB2, BB2, BB2)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(4), DD0, DD0, DD0)
	VPALIGNR(Imm(4), DD1, DD1, DD1)
	VPALIGNR(Imm(4), DD2, DD2, DD2)
	DECQ(itr2)
	JNE(LabelRef("openAVX2320InnerCipherLoop"))

	chacha20Constants := chacha20Constants_DATA()
	VMOVDQA(chacha20Constants, TT0)
	VPADDD(TT0, AA0, AA0)
	VPADDD(TT0, AA1, AA1)
	VPADDD(TT0, AA2, AA2)
	VPADDD(TT1, BB0, BB0)
	VPADDD(TT1, BB1, BB1)
	VPADDD(TT1, BB2, BB2)
	VPADDD(TT2, CC0, CC0)
	VPADDD(TT2, CC1, CC1)
	VPADDD(TT2, CC2, CC2)
	avx2IncMask := avx2IncMask_DATA()
	VMOVDQA(avx2IncMask, TT0)
	VPADDD(TT3, DD0, DD0)
	VPADDD(TT0, TT3, TT3)
	VPADDD(TT3, DD1, DD1)
	VPADDD(TT0, TT3, TT3)
	VPADDD(TT3, DD2, DD2)

	Comment("Clamp and store poly key")
	VPERM2I128(Imm(0x02), AA0, BB0, TT0)
	polyClampMask := polyClampMask_DATA()
	VPAND(polyClampMask, TT0, TT0)
	VMOVDQA(TT0, rsStoreAVX2)

	Comment("Stream for up to 320 bytes")
	VPERM2I128(Imm(0x13), AA0, BB0, AA0)
	VPERM2I128(Imm(0x13), CC0, DD0, BB0)
	VPERM2I128(Imm(0x02), AA1, BB1, CC0)
	VPERM2I128(Imm(0x02), CC1, DD1, DD0)
	VPERM2I128(Imm(0x13), AA1, BB1, AA1)
	VPERM2I128(Imm(0x13), CC1, DD1, BB1)
	VPERM2I128(Imm(0x02), AA2, BB2, CC1)
	VPERM2I128(Imm(0x02), CC2, DD2, DD1)
	VPERM2I128(Imm(0x13), AA2, BB2, AA2)
	VPERM2I128(Imm(0x13), CC2, DD2, BB2)
	JMP(LabelRef("openAVX2ShortOpen"))
}

// ----------------------------------------------------------------------------
// Special optimization for the last 128 bytes of ciphertext

// Need to decrypt up to 128 bytes - prepare two blocks
func openAVX2Tail128() {
	Label("openAVX2Tail128")
	Comment("Need to decrypt up to 128 bytes - prepare two blocks")
	chacha20Constants := chacha20Constants_DATA()
	VMOVDQA(chacha20Constants, AA1)
	VMOVDQA(state1StoreAVX2, BB1)
	VMOVDQA(state2StoreAVX2, CC1)
	VMOVDQA(ctr3StoreAVX2, DD1)
	avx2IncMask := avx2IncMask_DATA()
	VPADDD(avx2IncMask, DD1, DD1)
	VMOVDQA(DD1, DD0)

	XORQ(itr2, itr2)
	MOVQ(inl, itr1)
	ANDQ(I8(-16), itr1)
	TESTQ(itr1, itr1)
	JE(LabelRef("openAVX2Tail128LoopB"))
}

// Perform ChaCha rounds, while hashing the remaining input
func openAVX2Tail128LoopA() {
	Label("openAVX2Tail128LoopA")
	polyAdd(Mem{Base: inp, Index: itr2, Scale: 1}.Offset(0))
	polyMulAVX2()
}

func openAVX2Tail128LoopB() {
	Label("openAVX2Tail128LoopB")
	ADDQ(Imm(16), itr2)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	VPALIGNR(Imm(4), BB1, BB1, BB1)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(12), DD1, DD1, DD1)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	VPALIGNR(Imm(12), BB1, BB1, BB1)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(4), DD1, DD1, DD1)
	CMPQ(itr2, itr1)
	JB(LabelRef("openAVX2Tail128LoopA"))
	CMPQ(itr2, Imm(160))
	JNE(LabelRef("openAVX2Tail128LoopB"))

	chacha20Constants := chacha20Constants_DATA()
	VPADDD(chacha20Constants, AA1, AA1)
	VPADDD(state1StoreAVX2, BB1, BB1)
	VPADDD(state2StoreAVX2, CC1, CC1)
	VPADDD(DD0, DD1, DD1)
	VPERM2I128(Imm(0x02), AA1, BB1, AA0)
	VPERM2I128(Imm(0x02), CC1, DD1, BB0)
	VPERM2I128(Imm(0x13), AA1, BB1, CC0)
	VPERM2I128(Imm(0x13), CC1, DD1, DD0)
}

func openAVX2TailLoop() {
	Label("openAVX2TailLoop")
	CMPQ(inl, Imm(32))
	JB(LabelRef("openAVX2Tail"))
	SUBQ(Imm(32), inl)

	Comment("Load for decryption")
	VPXOR(Mem{Base: inp}, AA0, AA0)
	VMOVDQU(AA0, Mem{Base: oup})
	LEAQ(Mem{Base: inp}.Offset(1*32), inp)
	LEAQ(Mem{Base: oup}.Offset(1*32), oup)
	VMOVDQA(BB0, AA0)
	VMOVDQA(CC0, BB0)
	VMOVDQA(DD0, CC0)
	JMP(LabelRef("openAVX2TailLoop"))
}

func openAVX2Tail() {
	Label("openAVX2Tail")
	CMPQ(inl, Imm(16))
	VMOVDQA(A0, A1)
	JB(LabelRef("openAVX2TailDone"))
	SUBQ(Imm(16), inl)

	Comment("Load for decryption")
	VPXOR(Mem{Base: inp}, A0, T0)
	VMOVDQU(T0, Mem{Base: oup})
	LEAQ(Mem{Base: inp}.Offset(1*16), inp)
	LEAQ(Mem{Base: oup}.Offset(1*16), oup)
	VPERM2I128(Imm(0x11), AA0, AA0, AA0)
	VMOVDQA(A0, A1)
}

func openAVX2TailDone() {
	Label("openAVX2TailDone")
	VZEROUPPER()
	JMP(LabelRef("openSSETail16"))
}

// ----------------------------------------------------------------------------
// Special optimization for the last 256 bytes of ciphertext

// Need to decrypt up to 256 bytes - prepare four blocks
func openAVX2Tail256() {
	Label("openAVX2Tail256")
	chacha20Constants := chacha20Constants_DATA()
	VMOVDQA(chacha20Constants, AA0)
	VMOVDQA(AA0, AA1)
	VMOVDQA(state1StoreAVX2, BB0)
	VMOVDQA(BB0, BB1)
	VMOVDQA(state2StoreAVX2, CC0)
	VMOVDQA(CC0, CC1)
	VMOVDQA(ctr3StoreAVX2, DD0)
	avx2IncMask := avx2IncMask_DATA()
	VPADDD(avx2IncMask, DD0, DD0)
	VPADDD(avx2IncMask, DD0, DD1)
	VMOVDQA(DD0, TT1)
	VMOVDQA(DD1, TT2)

	Comment("Compute the number of iterations that will hash data")
	MOVQ(inl, tmpStoreAVX2)
	MOVQ(inl, itr1)
	SUBQ(Imm(128), itr1)
	SHRQ(Imm(4), itr1)
	MOVQ(U32(10), itr2)
	CMPQ(itr1, Imm(10))
	CMOVQGT(itr2, itr1)
	MOVQ(inp, inl)
	XORQ(itr2, itr2)
}

func openAVX2Tail256LoopA() {
	Label("openAVX2Tail256LoopA")
	polyAdd(Mem{Base: inl}.Offset(0))
	polyMulAVX2()
	LEAQ(Mem{Base: inl}.Offset(16), inl)
}

// Perform ChaCha rounds, while hashing the remaining input
func openAVX2Tail256LoopB() {
	Label("openAVX2Tail256LoopB")
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	VPALIGNR(Imm(4), BB0, BB0, BB0)
	VPALIGNR(Imm(4), BB1, BB1, BB1)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(12), DD0, DD0, DD0)
	VPALIGNR(Imm(12), DD1, DD1, DD1)
	INCQ(itr2)
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	VPALIGNR(Imm(12), BB0, BB0, BB0)
	VPALIGNR(Imm(12), BB1, BB1, BB1)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(4), DD0, DD0, DD0)
	VPALIGNR(Imm(4), DD1, DD1, DD1)
	CMPQ(itr2, itr1)
	JB(LabelRef("openAVX2Tail256LoopA"))

	CMPQ(itr2, Imm(10))
	JNE(LabelRef("openAVX2Tail256LoopB"))

	MOVQ(inl, itr2)
	SUBQ(inp, inl)
	MOVQ(inl, itr1)
	MOVQ(tmpStoreAVX2, inl)
}

// Hash the remainder of data (if any)
func openAVX2Tail256Hash() {
	Label("openAVX2Tail256Hash")
	ADDQ(Imm(16), itr1)
	CMPQ(itr1, inl)
	JGT(LabelRef("openAVX2Tail256HashEnd"))
	polyAdd(Mem{Base: itr2}.Offset(0))
	polyMulAVX2()
	LEAQ(Mem{Base: itr2}.Offset(16), itr2)
	JMP(LabelRef("openAVX2Tail256Hash"))
}

// Store 128 bytes safely, then go to store loop
func openAVX2Tail256HashEnd() {
	Label("openAVX2Tail256HashEnd")
	chacha20Constants := chacha20Constants_DATA()
	VPADDD(chacha20Constants, AA0, AA0)
	VPADDD(chacha20Constants, AA1, AA1)
	VPADDD(state1StoreAVX2, BB0, BB0)
	VPADDD(state1StoreAVX2, BB1, BB1)
	VPADDD(state2StoreAVX2, CC0, CC0)
	VPADDD(state2StoreAVX2, CC1, CC1)
	VPADDD(TT1, DD0, DD0)
	VPADDD(TT2, DD1, DD1)
	VPERM2I128(Imm(0x02), AA0, BB0, AA2)
	VPERM2I128(Imm(0x02), CC0, DD0, BB2)
	VPERM2I128(Imm(0x13), AA0, BB0, CC2)
	VPERM2I128(Imm(0x13), CC0, DD0, DD2)
	VPERM2I128(Imm(0x02), AA1, BB1, AA0)
	VPERM2I128(Imm(0x02), CC1, DD1, BB0)
	VPERM2I128(Imm(0x13), AA1, BB1, CC0)
	VPERM2I128(Imm(0x13), CC1, DD1, DD0)

	VPXOR(Mem{Base: inp}.Offset(0*32), AA2, AA2)
	VPXOR(Mem{Base: inp}.Offset(1*32), BB2, BB2)
	VPXOR(Mem{Base: inp}.Offset(2*32), CC2, CC2)
	VPXOR(Mem{Base: inp}.Offset(3*32), DD2, DD2)
	VMOVDQU(AA2, Mem{Base: oup}.Offset(0*32))
	VMOVDQU(BB2, Mem{Base: oup}.Offset(1*32))
	VMOVDQU(CC2, Mem{Base: oup}.Offset(2*32))
	VMOVDQU(DD2, Mem{Base: oup}.Offset(3*32))
	LEAQ(Mem{Base: inp}.Offset(4*32), inp)
	LEAQ(Mem{Base: oup}.Offset(4*32), oup)
	SUBQ(Imm(4*32), inl)

	JMP(LabelRef("openAVX2TailLoop"))
}

// ----------------------------------------------------------------------------
// Special optimization for the last 384 bytes of ciphertext

// Need to decrypt up to 384 bytes - prepare six blocks
func openAVX2Tail384() {
	Label("openAVX2Tail384")
	Comment("Need to decrypt up to 384 bytes - prepare six blocks")
	chacha20Constants := chacha20Constants_DATA()
	VMOVDQA(chacha20Constants, AA0)
	VMOVDQA(AA0, AA1)
	VMOVDQA(AA0, AA2)
	VMOVDQA(state1StoreAVX2, BB0)
	VMOVDQA(BB0, BB1)
	VMOVDQA(BB0, BB2)
	VMOVDQA(state2StoreAVX2, CC0)
	VMOVDQA(CC0, CC1)
	VMOVDQA(CC0, CC2)
	VMOVDQA(ctr3StoreAVX2, DD0)
	avx2IncMask := avx2IncMask_DATA()
	VPADDD(avx2IncMask, DD0, DD0)
	VPADDD(avx2IncMask, DD0, DD1)
	VPADDD(avx2IncMask, DD1, DD2)
	VMOVDQA(DD0, ctr0StoreAVX2)
	VMOVDQA(DD1, ctr1StoreAVX2)
	VMOVDQA(DD2, ctr2StoreAVX2)

	Comment("Compute the number of iterations that will hash two blocks of data")
	MOVQ(inl, tmpStoreAVX2)
	MOVQ(inl, itr1)
	SUBQ(U32(256), itr1)
	SHRQ(Imm(4), itr1)
	ADDQ(Imm(6), itr1)
	MOVQ(U32(10), itr2)
	CMPQ(itr1, Imm(10))
	CMOVQGT(itr2, itr1)
	MOVQ(inp, inl)
	XORQ(itr2, itr2)
}

// Perform ChaCha rounds, while hashing the remaining input
func openAVX2Tail384LoopB() {
	Label("openAVX2Tail384LoopB")
	polyAdd(Mem{Base: inl}.Offset(0))
	polyMulAVX2()
	LEAQ(Mem{Base: inl}.Offset(16), inl)
}

func openAVX2Tail384LoopA() {
	Label("openAVX2Tail384LoopA")
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
	VPALIGNR(Imm(4), BB0, BB0, BB0)
	VPALIGNR(Imm(4), BB1, BB1, BB1)
	VPALIGNR(Imm(4), BB2, BB2, BB2)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(12), DD0, DD0, DD0)
	VPALIGNR(Imm(12), DD1, DD1, DD1)
	VPALIGNR(Imm(12), DD2, DD2, DD2)
	polyAdd(Mem{Base: inl}.Offset(0))
	polyMulAVX2()
	LEAQ(Mem{Base: inl}.Offset(16), inl)
	INCQ(itr2)
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
	VPALIGNR(Imm(12), BB0, BB0, BB0)
	VPALIGNR(Imm(12), BB1, BB1, BB1)
	VPALIGNR(Imm(12), BB2, BB2, BB2)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(4), DD0, DD0, DD0)
	VPALIGNR(Imm(4), DD1, DD1, DD1)
	VPALIGNR(Imm(4), DD2, DD2, DD2)

	CMPQ(itr2, itr1)
	JB(LabelRef("openAVX2Tail384LoopB"))

	CMPQ(itr2, Imm(10))
	JNE(LabelRef("openAVX2Tail384LoopA"))

	MOVQ(inl, itr2)
	SUBQ(inp, inl)
	MOVQ(inl, itr1)
	MOVQ(tmpStoreAVX2, inl)
}

func openAVX2Tail384Hash() {
	Label("openAVX2Tail384Hash")
	ADDQ(Imm(16), itr1)
	CMPQ(itr1, inl)
	JGT(LabelRef("openAVX2Tail384HashEnd"))
	polyAdd(Mem{Base: itr2}.Offset(0))
	polyMulAVX2()
	LEAQ(Mem{Base: itr2}.Offset(16), itr2)
	JMP(LabelRef("openAVX2Tail384Hash"))
}

// Store 256 bytes safely, then go to store loop
func openAVX2Tail384HashEnd() {
	Label("openAVX2Tail384HashEnd")
	chacha20Constants := chacha20Constants_DATA()
	VPADDD(chacha20Constants, AA0, AA0)
	VPADDD(chacha20Constants, AA1, AA1)
	VPADDD(chacha20Constants, AA2, AA2)
	VPADDD(state1StoreAVX2, BB0, BB0)
	VPADDD(state1StoreAVX2, BB1, BB1)
	VPADDD(state1StoreAVX2, BB2, BB2)
	VPADDD(state2StoreAVX2, CC0, CC0)
	VPADDD(state2StoreAVX2, CC1, CC1)
	VPADDD(state2StoreAVX2, CC2, CC2)
	VPADDD(ctr0StoreAVX2, DD0, DD0)
	VPADDD(ctr1StoreAVX2, DD1, DD1)
	VPADDD(ctr2StoreAVX2, DD2, DD2)
	VPERM2I128(Imm(0x02), AA0, BB0, TT0)
	VPERM2I128(Imm(0x02), CC0, DD0, TT1)
	VPERM2I128(Imm(0x13), AA0, BB0, TT2)
	VPERM2I128(Imm(0x13), CC0, DD0, TT3)
	VPXOR(Mem{Base: inp}.Offset(0*32), TT0, TT0)
	VPXOR(Mem{Base: inp}.Offset(1*32), TT1, TT1)
	VPXOR(Mem{Base: inp}.Offset(2*32), TT2, TT2)
	VPXOR(Mem{Base: inp}.Offset(3*32), TT3, TT3)
	VMOVDQU(TT0, Mem{Base: oup}.Offset(0*32))
	VMOVDQU(TT1, Mem{Base: oup}.Offset(1*32))
	VMOVDQU(TT2, Mem{Base: oup}.Offset(2*32))
	VMOVDQU(TT3, Mem{Base: oup}.Offset(3*32))
	VPERM2I128(Imm(0x02), AA1, BB1, TT0)
	VPERM2I128(Imm(0x02), CC1, DD1, TT1)
	VPERM2I128(Imm(0x13), AA1, BB1, TT2)
	VPERM2I128(Imm(0x13), CC1, DD1, TT3)
	VPXOR(Mem{Base: inp}.Offset(4*32), TT0, TT0)
	VPXOR(Mem{Base: inp}.Offset(5*32), TT1, TT1)
	VPXOR(Mem{Base: inp}.Offset(6*32), TT2, TT2)
	VPXOR(Mem{Base: inp}.Offset(7*32), TT3, TT3)
	VMOVDQU(TT0, Mem{Base: oup}.Offset(4*32))
	VMOVDQU(TT1, Mem{Base: oup}.Offset(5*32))
	VMOVDQU(TT2, Mem{Base: oup}.Offset(6*32))
	VMOVDQU(TT3, Mem{Base: oup}.Offset(7*32))
	VPERM2I128(Imm(0x02), AA2, BB2, AA0)
	VPERM2I128(Imm(0x02), CC2, DD2, BB0)
	VPERM2I128(Imm(0x13), AA2, BB2, CC0)
	VPERM2I128(Imm(0x13), CC2, DD2, DD0)
	LEAQ(Mem{Base: inp}.Offset(8*32), inp)
	LEAQ(Mem{Base: oup}.Offset(8*32), oup)
	SUBQ(U32(8*32), inl)
	JMP(LabelRef("openAVX2TailLoop"))
}

// ----------------------------------------------------------------------------
// Special optimization for the last 512 bytes of ciphertext

func openAVX2Tail512() {
	Label("openAVX2Tail512")
	chacha20Constants := chacha20Constants_DATA()
	VMOVDQU(chacha20Constants, AA0)
	VMOVDQA(AA0, AA1)
	VMOVDQA(AA0, AA2)
	VMOVDQA(AA0, AA3)
	VMOVDQA(state1StoreAVX2, BB0)
	VMOVDQA(BB0, BB1)
	VMOVDQA(BB0, BB2)
	VMOVDQA(BB0, BB3)
	VMOVDQA(state2StoreAVX2, CC0)
	VMOVDQA(CC0, CC1)
	VMOVDQA(CC0, CC2)
	VMOVDQA(CC0, CC3)
	VMOVDQA(ctr3StoreAVX2, DD0)
	avx2IncMask := avx2IncMask_DATA()
	VPADDD(avx2IncMask, DD0, DD0)
	VPADDD(avx2IncMask, DD0, DD1)
	VPADDD(avx2IncMask, DD1, DD2)
	VPADDD(avx2IncMask, DD2, DD3)
	VMOVDQA(DD0, ctr0StoreAVX2)
	VMOVDQA(DD1, ctr1StoreAVX2)
	VMOVDQA(DD2, ctr2StoreAVX2)
	VMOVDQA(DD3, ctr3StoreAVX2)
	XORQ(itr1, itr1)
	MOVQ(inp, itr2)
}

func openAVX2Tail512LoopB() {
	Label("openAVX2Tail512LoopB")
	polyAdd(Mem{Base: itr2}.Offset(0))
	polyMulAVX2()
	LEAQ(Mem{Base: itr2}.Offset(2*8), itr2)
}

func openAVX2Tail512LoopA() {
	Label("openAVX2Tail512LoopA")
	VPADDD(BB0, AA0, AA0)
	VPADDD(BB1, AA1, AA1)
	VPADDD(BB2, AA2, AA2)
	VPADDD(BB3, AA3, AA3)
	VPXOR(AA0, DD0, DD0)
	VPXOR(AA1, DD1, DD1)
	VPXOR(AA2, DD2, DD2)
	VPXOR(AA3, DD3, DD3)
	rol16 := rol16_DATA()
	VPSHUFB(rol16, DD0, DD0)
	VPSHUFB(rol16, DD1, DD1)
	VPSHUFB(rol16, DD2, DD2)
	VPSHUFB(rol16, DD3, DD3)
	VPADDD(DD0, CC0, CC0)
	VPADDD(DD1, CC1, CC1)
	VPADDD(DD2, CC2, CC2)
	VPADDD(DD3, CC3, CC3)
	VPXOR(CC0, BB0, BB0)
	VPXOR(CC1, BB1, BB1)
	VPXOR(CC2, BB2, BB2)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(CC3, tmpStoreAVX2)
	VPSLLD(Imm(12), BB0, CC3)
	VPSRLD(Imm(20), BB0, BB0)
	VPXOR(CC3, BB0, BB0)
	VPSLLD(Imm(12), BB1, CC3)
	VPSRLD(Imm(20), BB1, BB1)
	VPXOR(CC3, BB1, BB1)
	VPSLLD(Imm(12), BB2, CC3)
	VPSRLD(Imm(20), BB2, BB2)
	VPXOR(CC3, BB2, BB2)
	VPSLLD(Imm(12), BB3, CC3)
	VPSRLD(Imm(20), BB3, BB3)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(tmpStoreAVX2, CC3)
	polyAdd(Mem{Base: itr2}.Offset(0 * 8))
	polyMulAVX2()
	VPADDD(BB0, AA0, AA0)
	VPADDD(BB1, AA1, AA1)
	VPADDD(BB2, AA2, AA2)
	VPADDD(BB3, AA3, AA3)
	VPXOR(AA0, DD0, DD0)
	VPXOR(AA1, DD1, DD1)
	VPXOR(AA2, DD2, DD2)
	VPXOR(AA3, DD3, DD3)
	rol8 := rol8_DATA()
	VPSHUFB(rol8, DD0, DD0)
	VPSHUFB(rol8, DD1, DD1)
	VPSHUFB(rol8, DD2, DD2)
	VPSHUFB(rol8, DD3, DD3)
	VPADDD(DD0, CC0, CC0)
	VPADDD(DD1, CC1, CC1)
	VPADDD(DD2, CC2, CC2)
	VPADDD(DD3, CC3, CC3)
	VPXOR(CC0, BB0, BB0)
	VPXOR(CC1, BB1, BB1)
	VPXOR(CC2, BB2, BB2)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(CC3, tmpStoreAVX2)
	VPSLLD(Imm(7), BB0, CC3)
	VPSRLD(Imm(25), BB0, BB0)
	VPXOR(CC3, BB0, BB0)
	VPSLLD(Imm(7), BB1, CC3)
	VPSRLD(Imm(25), BB1, BB1)
	VPXOR(CC3, BB1, BB1)
	VPSLLD(Imm(7), BB2, CC3)
	VPSRLD(Imm(25), BB2, BB2)
	VPXOR(CC3, BB2, BB2)
	VPSLLD(Imm(7), BB3, CC3)
	VPSRLD(Imm(25), BB3, BB3)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(tmpStoreAVX2, CC3)
	VPALIGNR(Imm(4), BB0, BB0, BB0)
	VPALIGNR(Imm(4), BB1, BB1, BB1)
	VPALIGNR(Imm(4), BB2, BB2, BB2)
	VPALIGNR(Imm(4), BB3, BB3, BB3)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(8), CC3, CC3, CC3)
	VPALIGNR(Imm(12), DD0, DD0, DD0)
	VPALIGNR(Imm(12), DD1, DD1, DD1)
	VPALIGNR(Imm(12), DD2, DD2, DD2)
	VPALIGNR(Imm(12), DD3, DD3, DD3)
	VPADDD(BB0, AA0, AA0)
	VPADDD(BB1, AA1, AA1)
	VPADDD(BB2, AA2, AA2)
	VPADDD(BB3, AA3, AA3)
	VPXOR(AA0, DD0, DD0)
	VPXOR(AA1, DD1, DD1)
	VPXOR(AA2, DD2, DD2)
	VPXOR(AA3, DD3, DD3)
	VPSHUFB(rol16, DD0, DD0)
	VPSHUFB(rol16, DD1, DD1)
	VPSHUFB(rol16, DD2, DD2)
	VPSHUFB(rol16, DD3, DD3)
	VPADDD(DD0, CC0, CC0)
	VPADDD(DD1, CC1, CC1)
	VPADDD(DD2, CC2, CC2)
	VPADDD(DD3, CC3, CC3)
	VPXOR(CC0, BB0, BB0)
	VPXOR(CC1, BB1, BB1)
	VPXOR(CC2, BB2, BB2)
	VPXOR(CC3, BB3, BB3)
	polyAdd(Mem{Base: itr2}.Offset(2 * 8))
	polyMulAVX2()
	LEAQ(Mem{Base: itr2}.Offset(4*8), itr2)
	VMOVDQA(CC3, tmpStoreAVX2)
	VPSLLD(Imm(12), BB0, CC3)
	VPSRLD(Imm(20), BB0, BB0)
	VPXOR(CC3, BB0, BB0)
	VPSLLD(Imm(12), BB1, CC3)
	VPSRLD(Imm(20), BB1, BB1)
	VPXOR(CC3, BB1, BB1)
	VPSLLD(Imm(12), BB2, CC3)
	VPSRLD(Imm(20), BB2, BB2)
	VPXOR(CC3, BB2, BB2)
	VPSLLD(Imm(12), BB3, CC3)
	VPSRLD(Imm(20), BB3, BB3)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(tmpStoreAVX2, CC3)
	VPADDD(BB0, AA0, AA0)
	VPADDD(BB1, AA1, AA1)
	VPADDD(BB2, AA2, AA2)
	VPADDD(BB3, AA3, AA3)
	VPXOR(AA0, DD0, DD0)
	VPXOR(AA1, DD1, DD1)
	VPXOR(AA2, DD2, DD2)
	VPXOR(AA3, DD3, DD3)
	VPSHUFB(rol8, DD0, DD0)
	VPSHUFB(rol8, DD1, DD1)
	VPSHUFB(rol8, DD2, DD2)
	VPSHUFB(rol8, DD3, DD3)
	VPADDD(DD0, CC0, CC0)
	VPADDD(DD1, CC1, CC1)
	VPADDD(DD2, CC2, CC2)
	VPADDD(DD3, CC3, CC3)
	VPXOR(CC0, BB0, BB0)
	VPXOR(CC1, BB1, BB1)
	VPXOR(CC2, BB2, BB2)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(CC3, tmpStoreAVX2)
	VPSLLD(Imm(7), BB0, CC3)
	VPSRLD(Imm(25), BB0, BB0)
	VPXOR(CC3, BB0, BB0)
	VPSLLD(Imm(7), BB1, CC3)
	VPSRLD(Imm(25), BB1, BB1)
	VPXOR(CC3, BB1, BB1)
	VPSLLD(Imm(7), BB2, CC3)
	VPSRLD(Imm(25), BB2, BB2)
	VPXOR(CC3, BB2, BB2)
	VPSLLD(Imm(7), BB3, CC3)
	VPSRLD(Imm(25), BB3, BB3)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(tmpStoreAVX2, CC3)
	VPALIGNR(Imm(12), BB0, BB0, BB0)
	VPALIGNR(Imm(12), BB1, BB1, BB1)
	VPALIGNR(Imm(12), BB2, BB2, BB2)
	VPALIGNR(Imm(12), BB3, BB3, BB3)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(8), CC3, CC3, CC3)
	VPALIGNR(Imm(4), DD0, DD0, DD0)
	VPALIGNR(Imm(4), DD1, DD1, DD1)
	VPALIGNR(Imm(4), DD2, DD2, DD2)
	VPALIGNR(Imm(4), DD3, DD3, DD3)
	INCQ(itr1)
	CMPQ(itr1, Imm(4))
	JLT(LabelRef("openAVX2Tail512LoopB"))

	CMPQ(itr1, Imm(10))
	JNE(LabelRef("openAVX2Tail512LoopA"))

	MOVQ(inl, itr1)
	SUBQ(U32(384), itr1)
	ANDQ(I8(-16), itr1)
}

func openAVX2Tail512HashLoop() {
	Label("openAVX2Tail512HashLoop")
	TESTQ(itr1, itr1)
	JE(LabelRef("openAVX2Tail512HashEnd"))
	polyAdd(Mem{Base: itr2}.Offset(0))
	polyMulAVX2()
	LEAQ(Mem{Base: itr2}.Offset(16), itr2)
	SUBQ(Imm(16), itr1)
	JMP(LabelRef("openAVX2Tail512HashLoop"))
}

func openAVX2Tail512HashEnd() {
	Label("openAVX2Tail512HashEnd")
	chacha20Constants := chacha20Constants_DATA()
	VPADDD(chacha20Constants, AA0, AA0)
	VPADDD(chacha20Constants, AA1, AA1)
	VPADDD(chacha20Constants, AA2, AA2)
	VPADDD(chacha20Constants, AA3, AA3)
	VPADDD(state1StoreAVX2, BB0, BB0)
	VPADDD(state1StoreAVX2, BB1, BB1)
	VPADDD(state1StoreAVX2, BB2, BB2)
	VPADDD(state1StoreAVX2, BB3, BB3)
	VPADDD(state2StoreAVX2, CC0, CC0)
	VPADDD(state2StoreAVX2, CC1, CC1)
	VPADDD(state2StoreAVX2, CC2, CC2)
	VPADDD(state2StoreAVX2, CC3, CC3)
	VPADDD(ctr0StoreAVX2, DD0, DD0)
	VPADDD(ctr1StoreAVX2, DD1, DD1)
	VPADDD(ctr2StoreAVX2, DD2, DD2)
	VPADDD(ctr3StoreAVX2, DD3, DD3)
	VMOVDQA(CC3, tmpStoreAVX2)
	VPERM2I128(Imm(0x02), AA0, BB0, CC3)
	VPERM2I128(Imm(0x13), AA0, BB0, BB0)
	VPERM2I128(Imm(0x02), CC0, DD0, AA0)
	VPERM2I128(Imm(0x13), CC0, DD0, CC0)
	VPXOR(Mem{Base: inp}.Offset(0*32), CC3, CC3)
	VPXOR(Mem{Base: inp}.Offset(1*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(2*32), BB0, BB0)
	VPXOR(Mem{Base: inp}.Offset(3*32), CC0, CC0)
	VMOVDQU(CC3, Mem{Base: oup}.Offset(0*32))
	VMOVDQU(AA0, Mem{Base: oup}.Offset(1*32))
	VMOVDQU(BB0, Mem{Base: oup}.Offset(2*32))
	VMOVDQU(CC0, Mem{Base: oup}.Offset(3*32))
	VPERM2I128(Imm(0x02), AA1, BB1, AA0)
	VPERM2I128(Imm(0x02), CC1, DD1, BB0)
	VPERM2I128(Imm(0x13), AA1, BB1, CC0)
	VPERM2I128(Imm(0x13), CC1, DD1, DD0)
	VPXOR(Mem{Base: inp}.Offset(4*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(5*32), BB0, BB0)
	VPXOR(Mem{Base: inp}.Offset(6*32), CC0, CC0)
	VPXOR(Mem{Base: inp}.Offset(7*32), DD0, DD0)
	VMOVDQU(AA0, Mem{Base: oup}.Offset(4*32))
	VMOVDQU(BB0, Mem{Base: oup}.Offset(5*32))
	VMOVDQU(CC0, Mem{Base: oup}.Offset(6*32))
	VMOVDQU(DD0, Mem{Base: oup}.Offset(7*32))
	VPERM2I128(Imm(0x02), AA2, BB2, AA0)
	VPERM2I128(Imm(0x02), CC2, DD2, BB0)
	VPERM2I128(Imm(0x13), AA2, BB2, CC0)
	VPERM2I128(Imm(0x13), CC2, DD2, DD0)
	VPXOR(Mem{Base: inp}.Offset(8*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(9*32), BB0, BB0)
	VPXOR(Mem{Base: inp}.Offset(10*32), CC0, CC0)
	VPXOR(Mem{Base: inp}.Offset(11*32), DD0, DD0)
	VMOVDQU(AA0, Mem{Base: oup}.Offset(8*32))
	VMOVDQU(BB0, Mem{Base: oup}.Offset(9*32))
	VMOVDQU(CC0, Mem{Base: oup}.Offset(10*32))
	VMOVDQU(DD0, Mem{Base: oup}.Offset(11*32))
	VPERM2I128(Imm(0x02), AA3, BB3, AA0)
	VPERM2I128(Imm(0x02), tmpStoreAVX2, DD3, BB0)
	VPERM2I128(Imm(0x13), AA3, BB3, CC0)
	VPERM2I128(Imm(0x13), tmpStoreAVX2, DD3, DD0)

	LEAQ(Mem{Base: inp}.Offset(12*32), inp)
	LEAQ(Mem{Base: oup}.Offset(12*32), oup)
	SUBQ(U32(12*32), inl)

	JMP(LabelRef("openAVX2TailLoop"))
}

// ----------------------------------------------------------------------------
// ----------------------------------------------------------------------------

// Implements the following function signature:
//
//	func chacha20Poly1305Seal(dst []byte, key []uint32, src, ad []byte)
func chacha20Poly1305Seal() {
	Implement("chacha20Poly1305Seal")
	Attributes(0)
	AllocLocal(288)

	MOVQ(RSP, RBP)
	ADDQ(Imm(32), RBP)
	ANDQ(I32(-32), RBP)
	Load(Param("dst").Base(), oup)
	Load(Param("key").Base(), keyp)
	Load(Param("src").Base(), inp)
	Load(Param("src").Len(), inl)
	Load(Param("ad").Base(), adp)

	CMPB(Mem{Symbol: Symbol{Name: ThatPeskyUnicodeDot + "useAVX2"}, Base: StaticBase}, Imm(1))
	JE(LabelRef("chacha20Poly1305Seal_AVX2"))

	Comment("Special optimization, for very short buffers")
	CMPQ(inl, Imm(128))
	JBE(LabelRef("sealSSE128"))

	Comment("In the seal case - prepare the poly key + 3 blocks of stream in the first iteration")
	chacha20Constants := chacha20Constants_DATA()
	MOVOU(chacha20Constants, A0)
	MOVOU(Mem{Base: keyp}.Offset(1*16), B0)
	MOVOU(Mem{Base: keyp}.Offset(2*16), C0)
	MOVOU(Mem{Base: keyp}.Offset(3*16), D0)

	Comment("Store state on stack for future use")
	MOVO(B0, state1Store)
	MOVO(C0, state2Store)

	Comment("Load state, increment counter blocks")
	MOVO(A0, A1)
	MOVO(B0, B1)
	MOVO(C0, C1)
	MOVO(D0, D1)
	sseIncMask := sseIncMask_DATA()
	PADDL(sseIncMask, D1)
	MOVO(A1, A2)
	MOVO(B1, B2)
	MOVO(C1, C2)
	MOVO(D1, D2)
	PADDL(sseIncMask, D2)
	MOVO(A2, A3)
	MOVO(B2, B3)
	MOVO(C2, C3)
	MOVO(D2, D3)
	PADDL(sseIncMask, D3)

	Comment("Store counters")
	MOVO(D0, ctr0Store)
	MOVO(D1, ctr1Store)
	MOVO(D2, ctr2Store)
	MOVO(D3, ctr3Store)
	MOVQ(U32(10), itr2)

	sealSSEIntroLoop()
	sealSSEMainLoop()

	// ----------------------------------------------------------------------------
	// Special optimization for the last 64 bytes of plaintext
	sealSSETail64()
	sealSSETail64LoopA()
	sealSSETail64LoopB()

	// ----------------------------------------------------------------------------
	// Special optimization for the last 128 bytes of plaintext
	sealSSETail128()
	sealSSETail128LoopA()
	sealSSETail128LoopB()

	// ----------------------------------------------------------------------------
	// Special optimization for the last 192 bytes of plaintext
	sealSSETail192()
	sealSSETail192LoopA()
	sealSSETail192LoopB()

	// ----------------------------------------------------------------------------
	// Special seal optimization for buffers smaller than 129 bytes
	sealSSE128()
	sealSSE128SealHash()
	sealSSE128Seal()
	sealSSETail()
	sealSSETailLoadLoop()
	sealSSEFinalize()

	// ----------------------------------------------------------------------------
	// ------------------------- AVX2 Code ----------------------------------------
	chacha20Poly1305Seal_AVX2()
	sealAVX2IntroLoop()
	sealAVX2MainLoop()
	sealAVX2InternalLoop()
	sealAVX2InternalLoopStart()

	// ----------------------------------------------------------------------------
	// Special optimization for buffers smaller than 193 bytes
	seal192AVX2()
	sealAVX2192InnerCipherLoop()
	sealAVX2ShortSeal()
	sealAVX2SealHash()
	sealAVX2ShortSealLoop()
	sealAVX2ShortTail32()
	sealAVX2ShortDone()

	// ----------------------------------------------------------------------------
	// Special optimization for buffers smaller than 321 bytes
	seal320AVX2()
	sealAVX2320InnerCipherLoop()

	// ----------------------------------------------------------------------------
	// Special optimization for the last 128 bytes of ciphertext
	sealAVX2Tail128()
	sealAVX2Tail128LoopA()
	sealAVX2Tail128LoopB()

	// ----------------------------------------------------------------------------
	// Special optimization for the last 256 bytes of ciphertext
	sealAVX2Tail256()
	sealAVX2Tail256LoopA()
	sealAVX2Tail256LoopB()

	// ----------------------------------------------------------------------------
	// Special optimization for the last 384 bytes of ciphertext
	sealAVX2Tail384()
	sealAVX2Tail384LoopA()
	sealAVX2Tail384LoopB()

	// ----------------------------------------------------------------------------
	// Special optimization for the last 512 bytes of ciphertext
	sealAVX2Tail512()
	sealAVX2Tail512LoopA()
	sealAVX2Tail512LoopB()
}

func sealSSEIntroLoop() {
	Label("sealSSEIntroLoop")
	MOVO(C3, tmpStore)
	chachaQR(A0, B0, C0, D0, C3)
	chachaQR(A1, B1, C1, D1, C3)
	chachaQR(A2, B2, C2, D2, C3)
	MOVO(tmpStore, C3)
	MOVO(C1, tmpStore)
	chachaQR(A3, B3, C3, D3, C1)
	MOVO(tmpStore, C1)
	shiftB0Left()
	shiftB1Left()
	shiftB2Left()
	shiftB3Left()
	shiftC0Left()
	shiftC1Left()
	shiftC2Left()
	shiftC3Left()
	shiftD0Left()
	shiftD1Left()
	shiftD2Left()
	shiftD3Left()

	MOVO(C3, tmpStore)
	chachaQR(A0, B0, C0, D0, C3)
	chachaQR(A1, B1, C1, D1, C3)
	chachaQR(A2, B2, C2, D2, C3)
	MOVO(tmpStore, C3)
	MOVO(C1, tmpStore)
	chachaQR(A3, B3, C3, D3, C1)
	MOVO(tmpStore, C1)
	shiftB0Right()
	shiftB1Right()
	shiftB2Right()
	shiftB3Right()
	shiftC0Right()
	shiftC1Right()
	shiftC2Right()
	shiftC3Right()
	shiftD0Right()
	shiftD1Right()
	shiftD2Right()
	shiftD3Right()
	DECQ(itr2)
	JNE(LabelRef("sealSSEIntroLoop"))

	Comment("Add in the state")
	chacha20Constants := chacha20Constants_DATA()
	PADDD(chacha20Constants, A0)
	PADDD(chacha20Constants, A1)
	PADDD(chacha20Constants, A2)
	PADDD(chacha20Constants, A3)
	PADDD(state1Store, B0)
	PADDD(state1Store, B1)
	PADDD(state1Store, B2)
	PADDD(state1Store, B3)
	PADDD(state2Store, C1)
	PADDD(state2Store, C2)
	PADDD(state2Store, C3)
	PADDD(ctr1Store, D1)
	PADDD(ctr2Store, D2)
	PADDD(ctr3Store, D3)

	Comment("Clamp and store the key")
	polyClampMask := polyClampMask_DATA()
	PAND(polyClampMask, A0)
	MOVO(A0, rStore)
	MOVO(B0, sStore)

	Comment("Hash AAD")
	MOVQ(NewParamAddr("ad_len", 80), itr2)
	CALL(LabelRef("polyHashADInternal<>(SB)"))

	MOVOU(Mem{Base: inp}.Offset(0*16), A0)
	MOVOU(Mem{Base: inp}.Offset(1*16), B0)
	MOVOU(Mem{Base: inp}.Offset(2*16), C0)
	MOVOU(Mem{Base: inp}.Offset(3*16), D0)
	PXOR(A0, A1)
	PXOR(B0, B1)
	PXOR(C0, C1)
	PXOR(D0, D1)
	MOVOU(A1, Mem{Base: oup}.Offset(0*16))
	MOVOU(B1, Mem{Base: oup}.Offset(1*16))
	MOVOU(C1, Mem{Base: oup}.Offset(2*16))
	MOVOU(D1, Mem{Base: oup}.Offset(3*16))
	MOVOU(Mem{Base: inp}.Offset(4*16), A0)
	MOVOU(Mem{Base: inp}.Offset(5*16), B0)
	MOVOU(Mem{Base: inp}.Offset(6*16), C0)
	MOVOU(Mem{Base: inp}.Offset(7*16), D0)
	PXOR(A0, A2)
	PXOR(B0, B2)
	PXOR(C0, C2)
	PXOR(D0, D2)
	MOVOU(A2, Mem{Base: oup}.Offset(4*16))
	MOVOU(B2, Mem{Base: oup}.Offset(5*16))
	MOVOU(C2, Mem{Base: oup}.Offset(6*16))
	MOVOU(D2, Mem{Base: oup}.Offset(7*16))

	MOVQ(U32(128), itr1)
	SUBQ(Imm(128), inl)
	LEAQ(Mem{Base: inp}.Offset(128), inp)

	MOVO(A3, A1)
	MOVO(B3, B1)
	MOVO(C3, C1)
	MOVO(D3, D1)

	CMPQ(inl, Imm(64))
	JBE(LabelRef("sealSSE128SealHash"))

	MOVOU(Mem{Base: inp}.Offset(0*16), A0)
	MOVOU(Mem{Base: inp}.Offset(1*16), B0)
	MOVOU(Mem{Base: inp}.Offset(2*16), C0)
	MOVOU(Mem{Base: inp}.Offset(3*16), D0)
	PXOR(A0, A3)
	PXOR(B0, B3)
	PXOR(C0, C3)
	PXOR(D0, D3)
	MOVOU(A3, Mem{Base: oup}.Offset(8*16))
	MOVOU(B3, Mem{Base: oup}.Offset(9*16))
	MOVOU(C3, Mem{Base: oup}.Offset(10*16))
	MOVOU(D3, Mem{Base: oup}.Offset(11*16))

	ADDQ(Imm(64), itr1)
	SUBQ(Imm(64), inl)
	LEAQ(Mem{Base: inp}.Offset(64), inp)

	MOVQ(U32(2), itr1)
	MOVQ(U32(8), itr2)

	CMPQ(inl, Imm(64))
	JBE(LabelRef("sealSSETail64"))
	CMPQ(inl, Imm(128))
	JBE(LabelRef("sealSSETail128"))
	CMPQ(inl, Imm(192))
	JBE(LabelRef("sealSSETail192"))
}

func sealSSEMainLoop() {
	Label("sealSSEMainLoop")
	Comment("Load state, increment counter blocks")
	chacha20Constants := chacha20Constants_DATA()
	MOVO(chacha20Constants, A0)
	MOVO(state1Store, B0)
	MOVO(state2Store, C0)
	MOVO(ctr3Store, D0)
	sseIncMask := sseIncMask_DATA()
	PADDL(sseIncMask, D0)
	MOVO(A0, A1)
	MOVO(B0, B1)
	MOVO(C0, C1)
	MOVO(D0, D1)
	PADDL(sseIncMask, D1)
	MOVO(A1, A2)
	MOVO(B1, B2)
	MOVO(C1, C2)
	MOVO(D1, D2)
	PADDL(sseIncMask, D2)
	MOVO(A2, A3)
	MOVO(B2, B3)
	MOVO(C2, C3)
	MOVO(D2, D3)
	PADDL(sseIncMask, D3)

	Comment("Store counters")
	MOVO(D0, ctr0Store)
	MOVO(D1, ctr1Store)
	MOVO(D2, ctr2Store)
	MOVO(D3, ctr3Store)

	Label("sealSSEInnerLoop")
	MOVO(C3, tmpStore)
	chachaQR(A0, B0, C0, D0, C3)
	chachaQR(A1, B1, C1, D1, C3)
	chachaQR(A2, B2, C2, D2, C3)
	MOVO(tmpStore, C3)
	MOVO(C1, tmpStore)
	chachaQR(A3, B3, C3, D3, C1)
	MOVO(tmpStore, C1)
	polyAdd(Mem{Base: oup}.Offset(0))
	shiftB0Left()
	shiftB1Left()
	shiftB2Left()
	shiftB3Left()
	shiftC0Left()
	shiftC1Left()
	shiftC2Left()
	shiftC3Left()
	shiftD0Left()
	shiftD1Left()
	shiftD2Left()
	shiftD3Left()
	polyMulStage1()
	polyMulStage2()
	LEAQ(Mem{Base: oup}.Offset(2*8), oup)
	MOVO(C3, tmpStore)
	chachaQR(A0, B0, C0, D0, C3)
	chachaQR(A1, B1, C1, D1, C3)
	chachaQR(A2, B2, C2, D2, C3)
	MOVO(tmpStore, C3)
	MOVO(C1, tmpStore)
	polyMulStage3()
	chachaQR(A3, B3, C3, D3, C1)
	MOVO(tmpStore, C1)
	polyMulReduceStage()
	shiftB0Right()
	shiftB1Right()
	shiftB2Right()
	shiftB3Right()
	shiftC0Right()
	shiftC1Right()
	shiftC2Right()
	shiftC3Right()
	shiftD0Right()
	shiftD1Right()
	shiftD2Right()
	shiftD3Right()
	DECQ(itr2)
	JGE(LabelRef("sealSSEInnerLoop"))
	polyAdd(Mem{Base: oup}.Offset(0))
	polyMul()
	LEAQ(Mem{Base: oup}.Offset(2*8), oup)
	DECQ(itr1)
	JG(LabelRef("sealSSEInnerLoop"))

	Comment("Add in the state")
	PADDD(chacha20Constants, A0)
	PADDD(chacha20Constants, A1)
	PADDD(chacha20Constants, A2)
	PADDD(chacha20Constants, A3)
	PADDD(state1Store, B0)
	PADDD(state1Store, B1)
	PADDD(state1Store, B2)
	PADDD(state1Store, B3)
	PADDD(state2Store, C0)
	PADDD(state2Store, C1)
	PADDD(state2Store, C2)
	PADDD(state2Store, C3)
	PADDD(ctr0Store, D0)
	PADDD(ctr1Store, D1)
	PADDD(ctr2Store, D2)
	PADDD(ctr3Store, D3)
	MOVO(D3, tmpStore)

	Comment("Load - xor - store")
	MOVOU(Mem{Base: inp}.Offset(0*16), D3)
	PXOR(D3, A0)
	MOVOU(Mem{Base: inp}.Offset(1*16), D3)
	PXOR(D3, B0)
	MOVOU(Mem{Base: inp}.Offset(2*16), D3)
	PXOR(D3, C0)
	MOVOU(Mem{Base: inp}.Offset(3*16), D3)
	PXOR(D3, D0)
	MOVOU(A0, Mem{Base: oup}.Offset(0*16))
	MOVOU(B0, Mem{Base: oup}.Offset(1*16))
	MOVOU(C0, Mem{Base: oup}.Offset(2*16))
	MOVOU(D0, Mem{Base: oup}.Offset(3*16))
	MOVO(tmpStore, D3)

	MOVOU(Mem{Base: inp}.Offset(4*16), A0)
	MOVOU(Mem{Base: inp}.Offset(5*16), B0)
	MOVOU(Mem{Base: inp}.Offset(6*16), C0)
	MOVOU(Mem{Base: inp}.Offset(7*16), D0)
	PXOR(A0, A1)
	PXOR(B0, B1)
	PXOR(C0, C1)
	PXOR(D0, D1)
	MOVOU(A1, Mem{Base: oup}.Offset(4*16))
	MOVOU(B1, Mem{Base: oup}.Offset(5*16))
	MOVOU(C1, Mem{Base: oup}.Offset(6*16))
	MOVOU(D1, Mem{Base: oup}.Offset(7*16))
	MOVOU(Mem{Base: inp}.Offset(8*16), A0)
	MOVOU(Mem{Base: inp}.Offset(9*16), B0)
	MOVOU(Mem{Base: inp}.Offset(10*16), C0)
	MOVOU(Mem{Base: inp}.Offset(11*16), D0)
	PXOR(A0, A2)
	PXOR(B0, B2)
	PXOR(C0, C2)
	PXOR(D0, D2)
	MOVOU(A2, Mem{Base: oup}.Offset(8*16))
	MOVOU(B2, Mem{Base: oup}.Offset(9*16))
	MOVOU(C2, Mem{Base: oup}.Offset(10*16))
	MOVOU(D2, Mem{Base: oup}.Offset(11*16))
	ADDQ(Imm(192), inp)
	MOVQ(U32(192), itr1)
	SUBQ(Imm(192), inl)
	MOVO(A3, A1)
	MOVO(B3, B1)
	MOVO(C3, C1)
	MOVO(D3, D1)
	CMPQ(inl, Imm(64))
	JBE(LabelRef("sealSSE128SealHash"))
	MOVOU(Mem{Base: inp}.Offset(0*16), A0)
	MOVOU(Mem{Base: inp}.Offset(1*16), B0)
	MOVOU(Mem{Base: inp}.Offset(2*16), C0)
	MOVOU(Mem{Base: inp}.Offset(3*16), D0)
	PXOR(A0, A3)
	PXOR(B0, B3)
	PXOR(C0, C3)
	PXOR(D0, D3)
	MOVOU(A3, Mem{Base: oup}.Offset(12*16))
	MOVOU(B3, Mem{Base: oup}.Offset(13*16))
	MOVOU(C3, Mem{Base: oup}.Offset(14*16))
	MOVOU(D3, Mem{Base: oup}.Offset(15*16))
	LEAQ(Mem{Base: inp}.Offset(64), inp)
	SUBQ(Imm(64), inl)
	MOVQ(U32(6), itr1)
	MOVQ(U32(4), itr2)
	CMPQ(inl, Imm(192))
	JG(LabelRef("sealSSEMainLoop"))

	MOVQ(inl, itr1)
	TESTQ(inl, inl)
	JE(LabelRef("sealSSE128SealHash"))
	MOVQ(U32(6), itr1)
	CMPQ(inl, Imm(64))
	JBE(LabelRef("sealSSETail64"))
	CMPQ(inl, Imm(128))
	JBE(LabelRef("sealSSETail128"))
	JMP(LabelRef("sealSSETail192"))
}

// ----------------------------------------------------------------------------
// Special optimization for the last 64 bytes of plaintext

// Need to encrypt up to 64 bytes - prepare single block, hash 192 or 256 bytes
func sealSSETail64() {
	Label("sealSSETail64")
	chacha20Constants := chacha20Constants_DATA()
	MOVO(chacha20Constants, A1)
	MOVO(state1Store, B1)
	MOVO(state2Store, C1)
	MOVO(ctr3Store, D1)
	sseIncMask := sseIncMask_DATA()
	PADDL(sseIncMask, D1)
	MOVO(D1, ctr0Store)
}

// Perform ChaCha rounds, while hashing the previously encrypted ciphertext
func sealSSETail64LoopA() {
	Label("sealSSETail64LoopA")
	polyAdd(Mem{Base: oup}.Offset(0))
	polyMul()
	LEAQ(Mem{Base: oup}.Offset(16), oup)
}

func sealSSETail64LoopB() {
	Label("sealSSETail64LoopB")
	chachaQR(A1, B1, C1, D1, T1)
	shiftB1Left()
	shiftC1Left()
	shiftD1Left()
	chachaQR(A1, B1, C1, D1, T1)
	shiftB1Right()
	shiftC1Right()
	shiftD1Right()
	polyAdd(Mem{Base: oup}.Offset(0))
	polyMul()
	LEAQ(Mem{Base: oup}.Offset(16), oup)

	DECQ(itr1)
	JG(LabelRef("sealSSETail64LoopA"))

	DECQ(itr2)
	JGE(LabelRef("sealSSETail64LoopB"))
	chacha20Constants := chacha20Constants_DATA()
	PADDL(chacha20Constants, A1)
	PADDL(state1Store, B1)
	PADDL(state2Store, C1)
	PADDL(ctr0Store, D1)

	JMP(LabelRef("sealSSE128Seal"))
}

// ----------------------------------------------------------------------------
// Special optimization for the last 128 bytes of plaintext

// Need to encrypt up to 128 bytes - prepare two blocks, hash 192 or 256 bytes
func sealSSETail128() {
	Label("sealSSETail128")
	chacha20Constants := chacha20Constants_DATA()
	MOVO(chacha20Constants, A0)
	MOVO(state1Store, B0)
	MOVO(state2Store, C0)
	MOVO(ctr3Store, D0)
	sseIncMask := sseIncMask_DATA()
	PADDL(sseIncMask, D0)
	MOVO(D0, ctr0Store)
	MOVO(A0, A1)
	MOVO(B0, B1)
	MOVO(C0, C1)
	MOVO(D0, D1)
	PADDL(sseIncMask, D1)
	MOVO(D1, ctr1Store)
}

// Perform ChaCha rounds, while hashing the previously encrypted ciphertext
func sealSSETail128LoopA() {
	Label("sealSSETail128LoopA")
	polyAdd(Mem{Base: oup}.Offset(0))
	polyMul()
	LEAQ(Mem{Base: oup}.Offset(16), oup)
}

func sealSSETail128LoopB() {
	Label("sealSSETail128LoopB")
	chachaQR(A0, B0, C0, D0, T0)
	chachaQR(A1, B1, C1, D1, T0)
	shiftB0Left()
	shiftC0Left()
	shiftD0Left()
	shiftB1Left()
	shiftC1Left()
	shiftD1Left()
	polyAdd(Mem{Base: oup}.Offset(0))
	polyMul()
	LEAQ(Mem{Base: oup}.Offset(16), oup)
	chachaQR(A0, B0, C0, D0, T0)
	chachaQR(A1, B1, C1, D1, T0)
	shiftB0Right()
	shiftC0Right()
	shiftD0Right()
	shiftB1Right()
	shiftC1Right()
	shiftD1Right()

	DECQ(itr1)
	JG(LabelRef("sealSSETail128LoopA"))

	DECQ(itr2)
	JGE(LabelRef("sealSSETail128LoopB"))

	chacha20Constants := chacha20Constants_DATA()
	PADDL(chacha20Constants, A0)
	PADDL(chacha20Constants, A1)
	PADDL(state1Store, B0)
	PADDL(state1Store, B1)
	PADDL(state2Store, C0)
	PADDL(state2Store, C1)
	PADDL(ctr0Store, D0)
	PADDL(ctr1Store, D1)

	MOVOU(Mem{Base: inp}.Offset(0*16), T0)
	MOVOU(Mem{Base: inp}.Offset(1*16), T1)
	MOVOU(Mem{Base: inp}.Offset(2*16), T2)
	MOVOU(Mem{Base: inp}.Offset(3*16), T3)
	PXOR(T0, A0)
	PXOR(T1, B0)
	PXOR(T2, C0)
	PXOR(T3, D0)
	MOVOU(A0, Mem{Base: oup}.Offset(0*16))
	MOVOU(B0, Mem{Base: oup}.Offset(1*16))
	MOVOU(C0, Mem{Base: oup}.Offset(2*16))
	MOVOU(D0, Mem{Base: oup}.Offset(3*16))

	MOVQ(U32(64), itr1)
	LEAQ(Mem{Base: inp}.Offset(64), inp)
	SUBQ(Imm(64), inl)

	JMP(LabelRef("sealSSE128SealHash"))
}

// ----------------------------------------------------------------------------
// Special optimization for the last 192 bytes of plaintext

// Need to encrypt up to 192 bytes - prepare three blocks, hash 192 or 256 bytes
func sealSSETail192() {
	Label("sealSSETail192")
	chacha20Constants := chacha20Constants_DATA()
	MOVO(chacha20Constants, A0)
	MOVO(state1Store, B0)
	MOVO(state2Store, C0)
	MOVO(ctr3Store, D0)
	sseIncMask := sseIncMask_DATA()
	PADDL(sseIncMask, D0)
	MOVO(D0, ctr0Store)
	MOVO(A0, A1)
	MOVO(B0, B1)
	MOVO(C0, C1)
	MOVO(D0, D1)
	PADDL(sseIncMask, D1)
	MOVO(D1, ctr1Store)
	MOVO(A1, A2)
	MOVO(B1, B2)
	MOVO(C1, C2)
	MOVO(D1, D2)
	PADDL(sseIncMask, D2)
	MOVO(D2, ctr2Store)
}

// Perform ChaCha rounds, while hashing the previously encrypted ciphertext
func sealSSETail192LoopA() {
	Label("sealSSETail192LoopA")
	polyAdd(Mem{Base: oup}.Offset(0))
	polyMul()
	LEAQ(Mem{Base: oup}.Offset(16), oup)
}

func sealSSETail192LoopB() {
	Label("sealSSETail192LoopB")
	chachaQR(A0, B0, C0, D0, T0)
	chachaQR(A1, B1, C1, D1, T0)
	chachaQR(A2, B2, C2, D2, T0)
	shiftB0Left()
	shiftC0Left()
	shiftD0Left()
	shiftB1Left()
	shiftC1Left()
	shiftD1Left()
	shiftB2Left()
	shiftC2Left()
	shiftD2Left()

	polyAdd(Mem{Base: oup}.Offset(0))
	polyMul()
	LEAQ(Mem{Base: oup}.Offset(16), oup)

	chachaQR(A0, B0, C0, D0, T0)
	chachaQR(A1, B1, C1, D1, T0)
	chachaQR(A2, B2, C2, D2, T0)
	shiftB0Right()
	shiftC0Right()
	shiftD0Right()
	shiftB1Right()
	shiftC1Right()
	shiftD1Right()
	shiftB2Right()
	shiftC2Right()
	shiftD2Right()

	DECQ(itr1)
	JG(LabelRef("sealSSETail192LoopA"))

	DECQ(itr2)
	JGE(LabelRef("sealSSETail192LoopB"))

	chacha20Constants := chacha20Constants_DATA()
	PADDL(chacha20Constants, A0)
	PADDL(chacha20Constants, A1)
	PADDL(chacha20Constants, A2)
	PADDL(state1Store, B0)
	PADDL(state1Store, B1)
	PADDL(state1Store, B2)
	PADDL(state2Store, C0)
	PADDL(state2Store, C1)
	PADDL(state2Store, C2)
	PADDL(ctr0Store, D0)
	PADDL(ctr1Store, D1)
	PADDL(ctr2Store, D2)

	MOVOU(Mem{Base: inp}.Offset(0*16), T0)
	MOVOU(Mem{Base: inp}.Offset(1*16), T1)
	MOVOU(Mem{Base: inp}.Offset(2*16), T2)
	MOVOU(Mem{Base: inp}.Offset(3*16), T3)
	PXOR(T0, A0)
	PXOR(T1, B0)
	PXOR(T2, C0)
	PXOR(T3, D0)
	MOVOU(A0, Mem{Base: oup}.Offset(0*16))
	MOVOU(B0, Mem{Base: oup}.Offset(1*16))
	MOVOU(C0, Mem{Base: oup}.Offset(2*16))
	MOVOU(D0, Mem{Base: oup}.Offset(3*16))
	MOVOU(Mem{Base: inp}.Offset(4*16), T0)
	MOVOU(Mem{Base: inp}.Offset(5*16), T1)
	MOVOU(Mem{Base: inp}.Offset(6*16), T2)
	MOVOU(Mem{Base: inp}.Offset(7*16), T3)
	PXOR(T0, A1)
	PXOR(T1, B1)
	PXOR(T2, C1)
	PXOR(T3, D1)
	MOVOU(A1, Mem{Base: oup}.Offset(4*16))
	MOVOU(B1, Mem{Base: oup}.Offset(5*16))
	MOVOU(C1, Mem{Base: oup}.Offset(6*16))
	MOVOU(D1, Mem{Base: oup}.Offset(7*16))

	MOVO(A2, A1)
	MOVO(B2, B1)
	MOVO(C2, C1)
	MOVO(D2, D1)
	MOVQ(U32(128), itr1)
	LEAQ(Mem{Base: inp}.Offset(128), inp)
	SUBQ(Imm(128), inl)

	JMP(LabelRef("sealSSE128SealHash"))
}

// ----------------------------------------------------------------------------
// Special seal optimization for buffers smaller than 129 bytes

// For up to 128 bytes of ciphertext and 64 bytes for the poly key, we require to process three blocks
func sealSSE128() {
	Label("sealSSE128")
	chacha20Constants := chacha20Constants_DATA()
	MOVOU(chacha20Constants, A0)
	MOVOU(Mem{Base: keyp}.Offset(1*16), B0)
	MOVOU(Mem{Base: keyp}.Offset(2*16), C0)
	MOVOU(Mem{Base: keyp}.Offset(3*16), D0)
	MOVO(A0, A1)
	MOVO(B0, B1)
	MOVO(C0, C1)
	MOVO(D0, D1)
	sseIncMask := sseIncMask_DATA()
	PADDL(sseIncMask, D1)
	MOVO(A1, A2)
	MOVO(B1, B2)
	MOVO(C1, C2)
	MOVO(D1, D2)
	PADDL(sseIncMask, D2)
	MOVO(B0, T1)
	MOVO(C0, T2)
	MOVO(D1, T3)
	MOVQ(U32(10), itr2)

	Label("sealSSE128InnerCipherLoop")
	chachaQR(A0, B0, C0, D0, T0)
	chachaQR(A1, B1, C1, D1, T0)
	chachaQR(A2, B2, C2, D2, T0)
	shiftB0Left()
	shiftB1Left()
	shiftB2Left()
	shiftC0Left()
	shiftC1Left()
	shiftC2Left()
	shiftD0Left()
	shiftD1Left()
	shiftD2Left()
	chachaQR(A0, B0, C0, D0, T0)
	chachaQR(A1, B1, C1, D1, T0)
	chachaQR(A2, B2, C2, D2, T0)
	shiftB0Right()
	shiftB1Right()
	shiftB2Right()
	shiftC0Right()
	shiftC1Right()
	shiftC2Right()
	shiftD0Right()
	shiftD1Right()
	shiftD2Right()
	DECQ(itr2)
	JNE(LabelRef("sealSSE128InnerCipherLoop"))

	Comment("A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded")
	PADDL(chacha20Constants, A0)
	PADDL(chacha20Constants, A1)
	PADDL(chacha20Constants, A2)
	PADDL(T1, B0)
	PADDL(T1, B1)
	PADDL(T1, B2)
	PADDL(T2, C1)
	PADDL(T2, C2)
	PADDL(T3, D1)
	PADDL(sseIncMask, T3)
	PADDL(T3, D2)
	polyClampMask := polyClampMask_DATA()
	PAND(polyClampMask, A0)
	MOVOU(A0, rStore)
	MOVOU(B0, sStore)

	Comment("Hash")
	MOVQ(NewParamAddr("ad_len", 80), itr2)
	CALL(LabelRef("polyHashADInternal<>(SB)"))
	XORQ(itr1, itr1)
}

// itr1 holds the number of bytes encrypted but not yet hashed
func sealSSE128SealHash() {
	Label("sealSSE128SealHash")
	CMPQ(itr1, Imm(16))
	JB(LabelRef("sealSSE128Seal"))
	polyAdd(Mem{Base: oup}.Offset(0))
	polyMul()

	SUBQ(Imm(16), itr1)
	ADDQ(Imm(16), oup)

	JMP(LabelRef("sealSSE128SealHash"))
}

func sealSSE128Seal() {
	Label("sealSSE128Seal")
	CMPQ(inl, Imm(16))
	JB(LabelRef("sealSSETail"))
	SUBQ(Imm(16), inl)

	Comment("Load for decryption")
	MOVOU(Mem{Base: inp}, T0)
	PXOR(T0, A1)
	MOVOU(A1, Mem{Base: oup})
	LEAQ(Mem{Base: inp}.Offset(1*16), inp)
	LEAQ(Mem{Base: oup}.Offset(1*16), oup)

	Comment("Extract for hashing")
	MOVQ(A1, t0)
	PSRLDQ(Imm(8), A1)
	MOVQ(A1, t1)
	ADDQ(t0, acc0)
	ADCQ(t1, acc1)
	ADCQ(Imm(1), acc2)
	polyMul()

	Comment("Shift the stream \"left\"")
	MOVO(B1, A1)
	MOVO(C1, B1)
	MOVO(D1, C1)
	MOVO(A2, D1)
	MOVO(B2, A2)
	MOVO(C2, B2)
	MOVO(D2, C2)
	JMP(LabelRef("sealSSE128Seal"))
}

func sealSSETail() {
	Label("sealSSETail")
	TESTQ(inl, inl)
	JE(LabelRef("sealSSEFinalize"))

	Comment("We can only load the PT one byte at a time to avoid read after end of buffer")
	MOVQ(inl, itr2)
	SHLQ(Imm(4), itr2)
	andMask := andMask_DATA()
	LEAQ(andMask, t0)
	MOVQ(inl, itr1)
	LEAQ(Mem{Base: inp, Index: inl, Scale: 1}.Offset(-1), inp)
	XORQ(t2, t2)
	XORQ(t3, t3)
	XORQ(RAX, RAX)
}

func sealSSETailLoadLoop() {
	Label("sealSSETailLoadLoop")
	SHLQ(Imm(8), t2, t3)
	SHLQ(Imm(8), t2)
	// Hack to get Avo to emit:
	// 	MOVB (inp), AX
	Instruction(&ir.Instruction{Opcode: "MOVB", Operands: []Op{Mem{Base: inp}, AX}})
	XORQ(RAX, t2)
	LEAQ(Mem{Base: inp}.Offset(-1), inp)
	DECQ(itr1)
	JNE(LabelRef("sealSSETailLoadLoop"))
	MOVQ(t2, tmpStore.Offset(0))
	MOVQ(t3, tmpStore.Offset(8))
	PXOR(tmpStore.Offset(0), A1)
	MOVOU(A1, Mem{Base: oup})
	MOVOU(Mem{Base: t0, Index: itr2, Scale: 1}.Offset(-16), T0)
	PAND(T0, A1)
	MOVQ(A1, t0)
	PSRLDQ(Imm(8), A1)
	MOVQ(A1, t1)
	ADDQ(t0, acc0)
	ADCQ(t1, acc1)
	ADCQ(Imm(1), acc2)
	polyMul()

	ADDQ(inl, oup)
}

func sealSSEFinalize() {
	Label("sealSSEFinalize")
	Comment("Hash in the buffer lengths")
	ADDQ(NewParamAddr("ad_len", 80), acc0)
	ADCQ(NewParamAddr("src_len", 56), acc1)
	ADCQ(Imm(1), acc2)
	polyMul()

	Comment("Final reduce")
	MOVQ(acc0, t0)
	MOVQ(acc1, t1)
	MOVQ(acc2, t2)
	SUBQ(I8(-5), acc0)
	SBBQ(I8(-1), acc1)
	SBBQ(Imm(3), acc2)
	CMOVQCS(t0, acc0)
	CMOVQCS(t1, acc1)
	CMOVQCS(t2, acc2)

	Comment("Add in the \"s\" part of the key")
	ADDQ(sStore.Offset(0), acc0)
	ADCQ(sStore.Offset(8), acc1)

	Comment("Finally store the tag at the end of the message")
	MOVQ(acc0, Mem{Base: oup}.Offset(0*8))
	MOVQ(acc1, Mem{Base: oup}.Offset(1*8))
	RET()
}

// ----------------------------------------------------------------------------
// ------------------------- AVX2 Code ----------------------------------------

func chacha20Poly1305Seal_AVX2() {
	Label("chacha20Poly1305Seal_AVX2")
	VZEROUPPER()
	chacha20Constants := chacha20Constants_DATA()
	VMOVDQU(chacha20Constants, AA0)
	VBROADCASTI128_16_R8_YMM14()
	VBROADCASTI128_32_R8_YMM12()
	VBROADCASTI128_48_R8_YMM4()
	avx2InitMask := avx2InitMask_DATA()
	VPADDD(avx2InitMask, DD0, DD0)

	Comment("Special optimizations, for very short buffers")
	CMPQ(inl, U32(192))
	JBE(LabelRef("seal192AVX2"))
	CMPQ(inl, U32(320))
	JBE(LabelRef("seal320AVX2"))

	Comment("For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream")
	VMOVDQA(AA0, AA1)
	VMOVDQA(AA0, AA2)
	VMOVDQA(AA0, AA3)
	VMOVDQA(BB0, BB1)
	VMOVDQA(BB0, BB2)
	VMOVDQA(BB0, BB3)
	VMOVDQA(BB0, state1StoreAVX2)
	VMOVDQA(CC0, CC1)
	VMOVDQA(CC0, CC2)
	VMOVDQA(CC0, CC3)
	VMOVDQA(CC0, state2StoreAVX2)
	avx2IncMask := avx2IncMask_DATA()
	VPADDD(avx2IncMask, DD0, DD1)
	VMOVDQA(DD0, ctr0StoreAVX2)
	VPADDD(avx2IncMask, DD1, DD2)
	VMOVDQA(DD1, ctr1StoreAVX2)
	VPADDD(avx2IncMask, DD2, DD3)
	VMOVDQA(DD2, ctr2StoreAVX2)
	VMOVDQA(DD3, ctr3StoreAVX2)
	MOVQ(U32(10), itr2)
}

func sealAVX2IntroLoop() {
	Label("sealAVX2IntroLoop")
	VMOVDQA(CC3, tmpStoreAVX2)
	chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3)
	chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3)
	VMOVDQA(tmpStoreAVX2, CC3)
	VMOVDQA(CC1, tmpStoreAVX2)
	chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1)
	VMOVDQA(tmpStoreAVX2, CC1)

	VPALIGNR(Imm(4), BB0, BB0, BB0)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(12), DD0, DD0, DD0)
	VPALIGNR(Imm(4), BB1, BB1, BB1)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(12), DD1, DD1, DD1)
	VPALIGNR(Imm(4), BB2, BB2, BB2)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(12), DD2, DD2, DD2)
	VPALIGNR(Imm(4), BB3, BB3, BB3)
	VPALIGNR(Imm(8), CC3, CC3, CC3)
	VPALIGNR(Imm(12), DD3, DD3, DD3)

	VMOVDQA(CC3, tmpStoreAVX2)
	chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3)
	chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3)
	VMOVDQA(tmpStoreAVX2, CC3)
	VMOVDQA(CC1, tmpStoreAVX2)
	chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1)
	VMOVDQA(tmpStoreAVX2, CC1)

	VPALIGNR(Imm(12), BB0, BB0, BB0)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(4), DD0, DD0, DD0)
	VPALIGNR(Imm(12), BB1, BB1, BB1)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(4), DD1, DD1, DD1)
	VPALIGNR(Imm(12), BB2, BB2, BB2)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(4), DD2, DD2, DD2)
	VPALIGNR(Imm(12), BB3, BB3, BB3)
	VPALIGNR(Imm(8), CC3, CC3, CC3)
	VPALIGNR(Imm(4), DD3, DD3, DD3)
	DECQ(itr2)
	JNE(LabelRef("sealAVX2IntroLoop"))

	chacha20Constants := chacha20Constants_DATA()
	VPADDD(chacha20Constants, AA0, AA0)
	VPADDD(chacha20Constants, AA1, AA1)
	VPADDD(chacha20Constants, AA2, AA2)
	VPADDD(chacha20Constants, AA3, AA3)
	VPADDD(state1StoreAVX2, BB0, BB0)
	VPADDD(state1StoreAVX2, BB1, BB1)
	VPADDD(state1StoreAVX2, BB2, BB2)
	VPADDD(state1StoreAVX2, BB3, BB3)
	VPADDD(state2StoreAVX2, CC0, CC0)
	VPADDD(state2StoreAVX2, CC1, CC1)
	VPADDD(state2StoreAVX2, CC2, CC2)
	VPADDD(state2StoreAVX2, CC3, CC3)
	VPADDD(ctr0StoreAVX2, DD0, DD0)
	VPADDD(ctr1StoreAVX2, DD1, DD1)
	VPADDD(ctr2StoreAVX2, DD2, DD2)
	VPADDD(ctr3StoreAVX2, DD3, DD3)

	VPERM2I128(Imm(0x13), CC0, DD0, CC0)
	VPERM2I128(Imm(0x02), AA0, BB0, DD0)
	VPERM2I128(Imm(0x13), AA0, BB0, AA0)

	Comment("Clamp and store poly key")
	polyClampMask := polyClampMask_DATA()
	VPAND(polyClampMask, DD0, DD0)
	VMOVDQA(DD0, rsStoreAVX2)

	Comment("Hash AD")
	MOVQ(NewParamAddr("ad_len", 80), itr2)
	CALL(LabelRef("polyHashADInternal<>(SB)"))

	Comment("Can store at least 320 bytes")
	VPXOR(Mem{Base: inp}.Offset(0*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(1*32), CC0, CC0)
	VMOVDQU(AA0, Mem{Base: oup}.Offset(0*32))
	VMOVDQU(CC0, Mem{Base: oup}.Offset(1*32))

	VPERM2I128(Imm(0x02), AA1, BB1, AA0)
	VPERM2I128(Imm(0x02), CC1, DD1, BB0)
	VPERM2I128(Imm(0x13), AA1, BB1, CC0)
	VPERM2I128(Imm(0x13), CC1, DD1, DD0)
	VPXOR(Mem{Base: inp}.Offset(2*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(3*32), BB0, BB0)
	VPXOR(Mem{Base: inp}.Offset(4*32), CC0, CC0)
	VPXOR(Mem{Base: inp}.Offset(5*32), DD0, DD0)
	VMOVDQU(AA0, Mem{Base: oup}.Offset(2*32))
	VMOVDQU(BB0, Mem{Base: oup}.Offset(3*32))
	VMOVDQU(CC0, Mem{Base: oup}.Offset(4*32))
	VMOVDQU(DD0, Mem{Base: oup}.Offset(5*32))
	VPERM2I128(Imm(0x02), AA2, BB2, AA0)
	VPERM2I128(Imm(0x02), CC2, DD2, BB0)
	VPERM2I128(Imm(0x13), AA2, BB2, CC0)
	VPERM2I128(Imm(0x13), CC2, DD2, DD0)
	VPXOR(Mem{Base: inp}.Offset(6*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(7*32), BB0, BB0)
	VPXOR(Mem{Base: inp}.Offset(8*32), CC0, CC0)
	VPXOR(Mem{Base: inp}.Offset(9*32), DD0, DD0)
	VMOVDQU(AA0, Mem{Base: oup}.Offset(6*32))
	VMOVDQU(BB0, Mem{Base: oup}.Offset(7*32))
	VMOVDQU(CC0, Mem{Base: oup}.Offset(8*32))
	VMOVDQU(DD0, Mem{Base: oup}.Offset(9*32))

	MOVQ(U32(320), itr1)
	SUBQ(U32(320), inl)
	LEAQ(Mem{Base: inp}.Offset(320), inp)

	VPERM2I128(Imm(0x02), AA3, BB3, AA0)
	VPERM2I128(Imm(0x02), CC3, DD3, BB0)
	VPERM2I128(Imm(0x13), AA3, BB3, CC0)
	VPERM2I128(Imm(0x13), CC3, DD3, DD0)
	CMPQ(inl, Imm(128))
	JBE(LabelRef("sealAVX2SealHash"))

	VPXOR(Mem{Base: inp}.Offset(0*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(1*32), BB0, BB0)
	VPXOR(Mem{Base: inp}.Offset(2*32), CC0, CC0)
	VPXOR(Mem{Base: inp}.Offset(3*32), DD0, DD0)
	VMOVDQU(AA0, Mem{Base: oup}.Offset(10*32))
	VMOVDQU(BB0, Mem{Base: oup}.Offset(11*32))
	VMOVDQU(CC0, Mem{Base: oup}.Offset(12*32))
	VMOVDQU(DD0, Mem{Base: oup}.Offset(13*32))
	SUBQ(Imm(128), inl)
	LEAQ(Mem{Base: inp}.Offset(128), inp)

	MOVQ(U32(8), itr1)
	MOVQ(U32(2), itr2)

	CMPQ(inl, Imm(128))
	JBE(LabelRef("sealAVX2Tail128"))
	CMPQ(inl, U32(256))
	JBE(LabelRef("sealAVX2Tail256"))
	CMPQ(inl, U32(384))
	JBE(LabelRef("sealAVX2Tail384"))
	CMPQ(inl, U32(512))
	JBE(LabelRef("sealAVX2Tail512"))

	Comment("We have 448 bytes to hash, but main loop hashes 512 bytes at a time - perform some rounds, before the main loop")
	VMOVDQA(chacha20Constants, AA0)
	VMOVDQA(AA0, AA1)
	VMOVDQA(AA0, AA2)
	VMOVDQA(AA0, AA3)
	VMOVDQA(state1StoreAVX2, BB0)
	VMOVDQA(BB0, BB1)
	VMOVDQA(BB0, BB2)
	VMOVDQA(BB0, BB3)
	VMOVDQA(state2StoreAVX2, CC0)
	VMOVDQA(CC0, CC1)
	VMOVDQA(CC0, CC2)
	VMOVDQA(CC0, CC3)
	VMOVDQA(ctr3StoreAVX2, DD0)
	avx2IncMask := avx2IncMask_DATA()
	VPADDD(avx2IncMask, DD0, DD0)
	VPADDD(avx2IncMask, DD0, DD1)
	VPADDD(avx2IncMask, DD1, DD2)
	VPADDD(avx2IncMask, DD2, DD3)
	VMOVDQA(DD0, ctr0StoreAVX2)
	VMOVDQA(DD1, ctr1StoreAVX2)
	VMOVDQA(DD2, ctr2StoreAVX2)
	VMOVDQA(DD3, ctr3StoreAVX2)

	VMOVDQA(CC3, tmpStoreAVX2)
	chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3)
	chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3)
	VMOVDQA(tmpStoreAVX2, CC3)
	VMOVDQA(CC1, tmpStoreAVX2)
	chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1)
	VMOVDQA(tmpStoreAVX2, CC1)

	VPALIGNR(Imm(4), BB0, BB0, BB0)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(12), DD0, DD0, DD0)
	VPALIGNR(Imm(4), BB1, BB1, BB1)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(12), DD1, DD1, DD1)
	VPALIGNR(Imm(4), BB2, BB2, BB2)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(12), DD2, DD2, DD2)
	VPALIGNR(Imm(4), BB3, BB3, BB3)
	VPALIGNR(Imm(8), CC3, CC3, CC3)
	VPALIGNR(Imm(12), DD3, DD3, DD3)

	VMOVDQA(CC3, tmpStoreAVX2)
	chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3)
	chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3)
	VMOVDQA(tmpStoreAVX2, CC3)
	VMOVDQA(CC1, tmpStoreAVX2)
	chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1)
	VMOVDQA(tmpStoreAVX2, CC1)

	VPALIGNR(Imm(12), BB0, BB0, BB0)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(4), DD0, DD0, DD0)
	VPALIGNR(Imm(12), BB1, BB1, BB1)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(4), DD1, DD1, DD1)
	VPALIGNR(Imm(12), BB2, BB2, BB2)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(4), DD2, DD2, DD2)
	VPALIGNR(Imm(12), BB3, BB3, BB3)
	VPALIGNR(Imm(8), CC3, CC3, CC3)
	VPALIGNR(Imm(4), DD3, DD3, DD3)
	VPADDD(BB0, AA0, AA0)
	VPADDD(BB1, AA1, AA1)
	VPADDD(BB2, AA2, AA2)
	VPADDD(BB3, AA3, AA3)
	VPXOR(AA0, DD0, DD0)
	VPXOR(AA1, DD1, DD1)
	VPXOR(AA2, DD2, DD2)
	VPXOR(AA3, DD3, DD3)
	rol16 := rol16_DATA()
	VPSHUFB(rol16, DD0, DD0)
	VPSHUFB(rol16, DD1, DD1)
	VPSHUFB(rol16, DD2, DD2)
	VPSHUFB(rol16, DD3, DD3)
	VPADDD(DD0, CC0, CC0)
	VPADDD(DD1, CC1, CC1)
	VPADDD(DD2, CC2, CC2)
	VPADDD(DD3, CC3, CC3)
	VPXOR(CC0, BB0, BB0)
	VPXOR(CC1, BB1, BB1)
	VPXOR(CC2, BB2, BB2)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(CC3, tmpStoreAVX2)
	VPSLLD(Imm(12), BB0, CC3)
	VPSRLD(Imm(20), BB0, BB0)
	VPXOR(CC3, BB0, BB0)
	VPSLLD(Imm(12), BB1, CC3)
	VPSRLD(Imm(20), BB1, BB1)
	VPXOR(CC3, BB1, BB1)
	VPSLLD(Imm(12), BB2, CC3)
	VPSRLD(Imm(20), BB2, BB2)
	VPXOR(CC3, BB2, BB2)
	VPSLLD(Imm(12), BB3, CC3)
	VPSRLD(Imm(20), BB3, BB3)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(tmpStoreAVX2, CC3)

	SUBQ(Imm(16), oup) // Adjust the pointer
	MOVQ(U32(9), itr1)
	JMP(LabelRef("sealAVX2InternalLoopStart"))
}

// Load state, increment counter blocks, store the incremented counters
func sealAVX2MainLoop() {
	Label("sealAVX2MainLoop")
	chacha20Constants := chacha20Constants_DATA()
	VMOVDQU(chacha20Constants, AA0)
	VMOVDQA(AA0, AA1)
	VMOVDQA(AA0, AA2)
	VMOVDQA(AA0, AA3)
	VMOVDQA(state1StoreAVX2, BB0)
	VMOVDQA(BB0, BB1)
	VMOVDQA(BB0, BB2)
	VMOVDQA(BB0, BB3)
	VMOVDQA(state2StoreAVX2, CC0)
	VMOVDQA(CC0, CC1)
	VMOVDQA(CC0, CC2)
	VMOVDQA(CC0, CC3)
	VMOVDQA(ctr3StoreAVX2, DD0)
	avx2IncMask := avx2IncMask_DATA()
	VPADDD(avx2IncMask, DD0, DD0)
	VPADDD(avx2IncMask, DD0, DD1)
	VPADDD(avx2IncMask, DD1, DD2)
	VPADDD(avx2IncMask, DD2, DD3)
	VMOVDQA(DD0, ctr0StoreAVX2)
	VMOVDQA(DD1, ctr1StoreAVX2)
	VMOVDQA(DD2, ctr2StoreAVX2)
	VMOVDQA(DD3, ctr3StoreAVX2)
	MOVQ(U32(10), itr1)
}

func sealAVX2InternalLoop() {
	Label("sealAVX2InternalLoop")
	polyAdd(Mem{Base: oup}.Offset(0 * 8))
	VPADDD(BB0, AA0, AA0)
	VPADDD(BB1, AA1, AA1)
	VPADDD(BB2, AA2, AA2)
	VPADDD(BB3, AA3, AA3)
	polyMulStage1_AVX2()
	VPXOR(AA0, DD0, DD0)
	VPXOR(AA1, DD1, DD1)
	VPXOR(AA2, DD2, DD2)
	VPXOR(AA3, DD3, DD3)
	rol16 := rol16_DATA()
	VPSHUFB(rol16, DD0, DD0)
	VPSHUFB(rol16, DD1, DD1)
	VPSHUFB(rol16, DD2, DD2)
	VPSHUFB(rol16, DD3, DD3)
	polyMulStage2_AVX2()
	VPADDD(DD0, CC0, CC0)
	VPADDD(DD1, CC1, CC1)
	VPADDD(DD2, CC2, CC2)
	VPADDD(DD3, CC3, CC3)
	VPXOR(CC0, BB0, BB0)
	VPXOR(CC1, BB1, BB1)
	VPXOR(CC2, BB2, BB2)
	VPXOR(CC3, BB3, BB3)
	polyMulStage3_AVX2()
	VMOVDQA(CC3, tmpStoreAVX2)
	VPSLLD(Imm(12), BB0, CC3)
	VPSRLD(Imm(20), BB0, BB0)
	VPXOR(CC3, BB0, BB0)
	VPSLLD(Imm(12), BB1, CC3)
	VPSRLD(Imm(20), BB1, BB1)
	VPXOR(CC3, BB1, BB1)
	VPSLLD(Imm(12), BB2, CC3)
	VPSRLD(Imm(20), BB2, BB2)
	VPXOR(CC3, BB2, BB2)
	VPSLLD(Imm(12), BB3, CC3)
	VPSRLD(Imm(20), BB3, BB3)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(tmpStoreAVX2, CC3)
	polyMulReduceStage()
}

func sealAVX2InternalLoopStart() {
	Label("sealAVX2InternalLoopStart")
	VPADDD(BB0, AA0, AA0)
	VPADDD(BB1, AA1, AA1)
	VPADDD(BB2, AA2, AA2)
	VPADDD(BB3, AA3, AA3)
	VPXOR(AA0, DD0, DD0)
	VPXOR(AA1, DD1, DD1)
	VPXOR(AA2, DD2, DD2)
	VPXOR(AA3, DD3, DD3)
	rol8 := rol8_DATA()
	VPSHUFB(rol8, DD0, DD0)
	VPSHUFB(rol8, DD1, DD1)
	VPSHUFB(rol8, DD2, DD2)
	VPSHUFB(rol8, DD3, DD3)
	polyAdd(Mem{Base: oup}.Offset(2 * 8))
	VPADDD(DD0, CC0, CC0)
	VPADDD(DD1, CC1, CC1)
	VPADDD(DD2, CC2, CC2)
	VPADDD(DD3, CC3, CC3)
	polyMulStage1_AVX2()
	VPXOR(CC0, BB0, BB0)
	VPXOR(CC1, BB1, BB1)
	VPXOR(CC2, BB2, BB2)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(CC3, tmpStoreAVX2)
	VPSLLD(Imm(7), BB0, CC3)
	VPSRLD(Imm(25), BB0, BB0)
	VPXOR(CC3, BB0, BB0)
	VPSLLD(Imm(7), BB1, CC3)
	VPSRLD(Imm(25), BB1, BB1)
	VPXOR(CC3, BB1, BB1)
	VPSLLD(Imm(7), BB2, CC3)
	VPSRLD(Imm(25), BB2, BB2)
	VPXOR(CC3, BB2, BB2)
	VPSLLD(Imm(7), BB3, CC3)
	VPSRLD(Imm(25), BB3, BB3)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(tmpStoreAVX2, CC3)
	polyMulStage2_AVX2()
	VPALIGNR(Imm(4), BB0, BB0, BB0)
	VPALIGNR(Imm(4), BB1, BB1, BB1)
	VPALIGNR(Imm(4), BB2, BB2, BB2)
	VPALIGNR(Imm(4), BB3, BB3, BB3)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(8), CC3, CC3, CC3)
	VPALIGNR(Imm(12), DD0, DD0, DD0)
	VPALIGNR(Imm(12), DD1, DD1, DD1)
	VPALIGNR(Imm(12), DD2, DD2, DD2)
	VPALIGNR(Imm(12), DD3, DD3, DD3)
	VPADDD(BB0, AA0, AA0)
	VPADDD(BB1, AA1, AA1)
	VPADDD(BB2, AA2, AA2)
	VPADDD(BB3, AA3, AA3)
	polyMulStage3_AVX2()
	VPXOR(AA0, DD0, DD0)
	VPXOR(AA1, DD1, DD1)
	VPXOR(AA2, DD2, DD2)
	VPXOR(AA3, DD3, DD3)
	rol16 := rol16_DATA()
	VPSHUFB(rol16, DD0, DD0)
	VPSHUFB(rol16, DD1, DD1)
	VPSHUFB(rol16, DD2, DD2)
	VPSHUFB(rol16, DD3, DD3)
	polyMulReduceStage()
	VPADDD(DD0, CC0, CC0)
	VPADDD(DD1, CC1, CC1)
	VPADDD(DD2, CC2, CC2)
	VPADDD(DD3, CC3, CC3)
	VPXOR(CC0, BB0, BB0)
	VPXOR(CC1, BB1, BB1)
	VPXOR(CC2, BB2, BB2)
	VPXOR(CC3, BB3, BB3)
	polyAdd(Mem{Base: oup}.Offset(4 * 8))
	LEAQ(Mem{Base: oup}.Offset(6*8), oup)
	VMOVDQA(CC3, tmpStoreAVX2)
	VPSLLD(Imm(12), BB0, CC3)
	VPSRLD(Imm(20), BB0, BB0)
	VPXOR(CC3, BB0, BB0)
	VPSLLD(Imm(12), BB1, CC3)
	VPSRLD(Imm(20), BB1, BB1)
	VPXOR(CC3, BB1, BB1)
	VPSLLD(Imm(12), BB2, CC3)
	VPSRLD(Imm(20), BB2, BB2)
	VPXOR(CC3, BB2, BB2)
	VPSLLD(Imm(12), BB3, CC3)
	VPSRLD(Imm(20), BB3, BB3)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(tmpStoreAVX2, CC3)
	polyMulStage1_AVX2()
	VPADDD(BB0, AA0, AA0)
	VPADDD(BB1, AA1, AA1)
	VPADDD(BB2, AA2, AA2)
	VPADDD(BB3, AA3, AA3)
	VPXOR(AA0, DD0, DD0)
	VPXOR(AA1, DD1, DD1)
	VPXOR(AA2, DD2, DD2)
	VPXOR(AA3, DD3, DD3)
	polyMulStage2_AVX2()
	VPSHUFB(rol8, DD0, DD0)
	VPSHUFB(rol8, DD1, DD1)
	VPSHUFB(rol8, DD2, DD2)
	VPSHUFB(rol8, DD3, DD3)
	VPADDD(DD0, CC0, CC0)
	VPADDD(DD1, CC1, CC1)
	VPADDD(DD2, CC2, CC2)
	VPADDD(DD3, CC3, CC3)
	polyMulStage3_AVX2()
	VPXOR(CC0, BB0, BB0)
	VPXOR(CC1, BB1, BB1)
	VPXOR(CC2, BB2, BB2)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(CC3, tmpStoreAVX2)
	VPSLLD(Imm(7), BB0, CC3)
	VPSRLD(Imm(25), BB0, BB0)
	VPXOR(CC3, BB0, BB0)
	VPSLLD(Imm(7), BB1, CC3)
	VPSRLD(Imm(25), BB1, BB1)
	VPXOR(CC3, BB1, BB1)
	VPSLLD(Imm(7), BB2, CC3)
	VPSRLD(Imm(25), BB2, BB2)
	VPXOR(CC3, BB2, BB2)
	VPSLLD(Imm(7), BB3, CC3)
	VPSRLD(Imm(25), BB3, BB3)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(tmpStoreAVX2, CC3)
	polyMulReduceStage()
	VPALIGNR(Imm(12), BB0, BB0, BB0)
	VPALIGNR(Imm(12), BB1, BB1, BB1)
	VPALIGNR(Imm(12), BB2, BB2, BB2)
	VPALIGNR(Imm(12), BB3, BB3, BB3)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(8), CC3, CC3, CC3)
	VPALIGNR(Imm(4), DD0, DD0, DD0)
	VPALIGNR(Imm(4), DD1, DD1, DD1)
	VPALIGNR(Imm(4), DD2, DD2, DD2)
	VPALIGNR(Imm(4), DD3, DD3, DD3)
	DECQ(itr1)
	JNE(LabelRef("sealAVX2InternalLoop"))

	chacha20Constants := chacha20Constants_DATA()
	VPADDD(chacha20Constants, AA0, AA0)
	VPADDD(chacha20Constants, AA1, AA1)
	VPADDD(chacha20Constants, AA2, AA2)
	VPADDD(chacha20Constants, AA3, AA3)
	VPADDD(state1StoreAVX2, BB0, BB0)
	VPADDD(state1StoreAVX2, BB1, BB1)
	VPADDD(state1StoreAVX2, BB2, BB2)
	VPADDD(state1StoreAVX2, BB3, BB3)
	VPADDD(state2StoreAVX2, CC0, CC0)
	VPADDD(state2StoreAVX2, CC1, CC1)
	VPADDD(state2StoreAVX2, CC2, CC2)
	VPADDD(state2StoreAVX2, CC3, CC3)
	VPADDD(ctr0StoreAVX2, DD0, DD0)
	VPADDD(ctr1StoreAVX2, DD1, DD1)
	VPADDD(ctr2StoreAVX2, DD2, DD2)
	VPADDD(ctr3StoreAVX2, DD3, DD3)
	VMOVDQA(CC3, tmpStoreAVX2)

	Comment("We only hashed 480 of the 512 bytes available - hash the remaining 32 here")
	polyAdd(Mem{Base: oup}.Offset(0 * 8))
	polyMulAVX2()
	LEAQ(Mem{Base: oup}.Offset(4*8), oup)
	VPERM2I128(Imm(0x02), AA0, BB0, CC3)
	VPERM2I128(Imm(0x13), AA0, BB0, BB0)
	VPERM2I128(Imm(0x02), CC0, DD0, AA0)
	VPERM2I128(Imm(0x13), CC0, DD0, CC0)
	VPXOR(Mem{Base: inp}.Offset(0*32), CC3, CC3)
	VPXOR(Mem{Base: inp}.Offset(1*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(2*32), BB0, BB0)
	VPXOR(Mem{Base: inp}.Offset(3*32), CC0, CC0)
	VMOVDQU(CC3, Mem{Base: oup}.Offset(0*32))
	VMOVDQU(AA0, Mem{Base: oup}.Offset(1*32))
	VMOVDQU(BB0, Mem{Base: oup}.Offset(2*32))
	VMOVDQU(CC0, Mem{Base: oup}.Offset(3*32))
	VPERM2I128(Imm(0x02), AA1, BB1, AA0)
	VPERM2I128(Imm(0x02), CC1, DD1, BB0)
	VPERM2I128(Imm(0x13), AA1, BB1, CC0)
	VPERM2I128(Imm(0x13), CC1, DD1, DD0)
	VPXOR(Mem{Base: inp}.Offset(4*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(5*32), BB0, BB0)
	VPXOR(Mem{Base: inp}.Offset(6*32), CC0, CC0)
	VPXOR(Mem{Base: inp}.Offset(7*32), DD0, DD0)
	VMOVDQU(AA0, Mem{Base: oup}.Offset(4*32))
	VMOVDQU(BB0, Mem{Base: oup}.Offset(5*32))
	VMOVDQU(CC0, Mem{Base: oup}.Offset(6*32))
	VMOVDQU(DD0, Mem{Base: oup}.Offset(7*32))

	Comment("and here")
	polyAdd(Mem{Base: oup}.Offset(-2 * 8))
	polyMulAVX2()
	VPERM2I128(Imm(0x02), AA2, BB2, AA0)
	VPERM2I128(Imm(0x02), CC2, DD2, BB0)
	VPERM2I128(Imm(0x13), AA2, BB2, CC0)
	VPERM2I128(Imm(0x13), CC2, DD2, DD0)
	VPXOR(Mem{Base: inp}.Offset(8*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(9*32), BB0, BB0)
	VPXOR(Mem{Base: inp}.Offset(10*32), CC0, CC0)
	VPXOR(Mem{Base: inp}.Offset(11*32), DD0, DD0)
	VMOVDQU(AA0, Mem{Base: oup}.Offset(8*32))
	VMOVDQU(BB0, Mem{Base: oup}.Offset(9*32))
	VMOVDQU(CC0, Mem{Base: oup}.Offset(10*32))
	VMOVDQU(DD0, Mem{Base: oup}.Offset(11*32))
	VPERM2I128(Imm(0x02), AA3, BB3, AA0)
	VPERM2I128(Imm(0x02), tmpStoreAVX2, DD3, BB0)
	VPERM2I128(Imm(0x13), AA3, BB3, CC0)
	VPERM2I128(Imm(0x13), tmpStoreAVX2, DD3, DD0)
	VPXOR(Mem{Base: inp}.Offset(12*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(13*32), BB0, BB0)
	VPXOR(Mem{Base: inp}.Offset(14*32), CC0, CC0)
	VPXOR(Mem{Base: inp}.Offset(15*32), DD0, DD0)
	VMOVDQU(AA0, Mem{Base: oup}.Offset(12*32))
	VMOVDQU(BB0, Mem{Base: oup}.Offset(13*32))
	VMOVDQU(CC0, Mem{Base: oup}.Offset(14*32))
	VMOVDQU(DD0, Mem{Base: oup}.Offset(15*32))
	LEAQ(Mem{Base: inp}.Offset(32*16), inp)
	SUBQ(U32(32*16), inl)
	CMPQ(inl, U32(512))
	JG(LabelRef("sealAVX2MainLoop"))

	Comment("Tail can only hash 480 bytes")
	polyAdd(Mem{Base: oup}.Offset(0 * 8))
	polyMulAVX2()
	polyAdd(Mem{Base: oup}.Offset(2 * 8))
	polyMulAVX2()
	LEAQ(Mem{Base: oup}.Offset(32), oup)

	MOVQ(U32(10), itr1)
	MOVQ(U32(0), itr2)
	CMPQ(inl, Imm(128))
	JBE(LabelRef("sealAVX2Tail128"))
	CMPQ(inl, U32(256))
	JBE(LabelRef("sealAVX2Tail256"))
	CMPQ(inl, U32(384))
	JBE(LabelRef("sealAVX2Tail384"))
	JMP(LabelRef("sealAVX2Tail512"))
}

// ----------------------------------------------------------------------------
// Special optimization for buffers smaller than 193 bytes

// For up to 192 bytes of ciphertext and 64 bytes for the poly key, we process four blocks
func seal192AVX2() {
	Label("seal192AVX2")
	VMOVDQA(AA0, AA1)
	VMOVDQA(BB0, BB1)
	VMOVDQA(CC0, CC1)
	avx2IncMask := avx2IncMask_DATA()
	VPADDD(avx2IncMask, DD0, DD1)
	VMOVDQA(AA0, AA2)
	VMOVDQA(BB0, BB2)
	VMOVDQA(CC0, CC2)
	VMOVDQA(DD0, DD2)
	VMOVDQA(DD1, TT3)
	MOVQ(U32(10), itr2)
}

func sealAVX2192InnerCipherLoop() {
	Label("sealAVX2192InnerCipherLoop")
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	VPALIGNR(Imm(4), BB0, BB0, BB0)
	VPALIGNR(Imm(4), BB1, BB1, BB1)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(12), DD0, DD0, DD0)
	VPALIGNR(Imm(12), DD1, DD1, DD1)
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	VPALIGNR(Imm(12), BB0, BB0, BB0)
	VPALIGNR(Imm(12), BB1, BB1, BB1)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(4), DD0, DD0, DD0)
	VPALIGNR(Imm(4), DD1, DD1, DD1)
	DECQ(itr2)
	JNE(LabelRef("sealAVX2192InnerCipherLoop"))
	VPADDD(AA2, AA0, AA0)
	VPADDD(AA2, AA1, AA1)
	VPADDD(BB2, BB0, BB0)
	VPADDD(BB2, BB1, BB1)
	VPADDD(CC2, CC0, CC0)
	VPADDD(CC2, CC1, CC1)
	VPADDD(DD2, DD0, DD0)
	VPADDD(TT3, DD1, DD1)
	VPERM2I128(Imm(0x02), AA0, BB0, TT0)

	Comment("Clamp and store poly key")
	polyClampMask := polyClampMask_DATA()
	VPAND(polyClampMask, TT0, TT0)
	VMOVDQA(TT0, rsStoreAVX2)

	Comment("Stream for up to 192 bytes")
	VPERM2I128(Imm(0x13), AA0, BB0, AA0)
	VPERM2I128(Imm(0x13), CC0, DD0, BB0)
	VPERM2I128(Imm(0x02), AA1, BB1, CC0)
	VPERM2I128(Imm(0x02), CC1, DD1, DD0)
	VPERM2I128(Imm(0x13), AA1, BB1, AA1)
	VPERM2I128(Imm(0x13), CC1, DD1, BB1)
}

func sealAVX2ShortSeal() {
	Label("sealAVX2ShortSeal")
	Comment("Hash aad")
	MOVQ(NewParamAddr("ad_len", 80), itr2)
	CALL(LabelRef("polyHashADInternal<>(SB)"))
	XORQ(itr1, itr1)
}

func sealAVX2SealHash() {
	Label("sealAVX2SealHash")
	Comment("itr1 holds the number of bytes encrypted but not yet hashed")
	CMPQ(itr1, Imm(16))
	JB(LabelRef("sealAVX2ShortSealLoop"))
	polyAdd(Mem{Base: oup}.Offset(0))
	polyMul()
	SUBQ(Imm(16), itr1)
	ADDQ(Imm(16), oup)
	JMP(LabelRef("sealAVX2SealHash"))
}

func sealAVX2ShortSealLoop() {
	Label("sealAVX2ShortSealLoop")
	CMPQ(inl, Imm(32))
	JB(LabelRef("sealAVX2ShortTail32"))
	SUBQ(Imm(32), inl)

	Comment("Load for encryption")
	VPXOR(Mem{Base: inp}, AA0, AA0)
	VMOVDQU(AA0, Mem{Base: oup})
	LEAQ(Mem{Base: inp}.Offset(1*32), inp)

	Comment("Now can hash")
	polyAdd(Mem{Base: oup}.Offset(0 * 8))
	polyMulAVX2()
	polyAdd(Mem{Base: oup}.Offset(2 * 8))
	polyMulAVX2()
	LEAQ(Mem{Base: oup}.Offset(1*32), oup)

	Comment("Shift stream left")
	VMOVDQA(BB0, AA0)
	VMOVDQA(CC0, BB0)
	VMOVDQA(DD0, CC0)
	VMOVDQA(AA1, DD0)
	VMOVDQA(BB1, AA1)
	VMOVDQA(CC1, BB1)
	VMOVDQA(DD1, CC1)
	VMOVDQA(AA2, DD1)
	VMOVDQA(BB2, AA2)
	JMP(LabelRef("sealAVX2ShortSealLoop"))
}

func sealAVX2ShortTail32() {
	Label("sealAVX2ShortTail32")
	CMPQ(inl, Imm(16))
	VMOVDQA(A0, A1)
	JB(LabelRef("sealAVX2ShortDone"))

	SUBQ(Imm(16), inl)

	Comment("Load for encryption")
	VPXOR(Mem{Base: inp}, A0, T0)
	VMOVDQU(T0, Mem{Base: oup})
	LEAQ(Mem{Base: inp}.Offset(1*16), inp)

	Comment("Hash")
	polyAdd(Mem{Base: oup}.Offset(0 * 8))
	polyMulAVX2()
	LEAQ(Mem{Base: oup}.Offset(1*16), oup)
	VPERM2I128(Imm(0x11), AA0, AA0, AA0)
	VMOVDQA(A0, A1)
}

func sealAVX2ShortDone() {
	Label("sealAVX2ShortDone")
	VZEROUPPER()
	JMP(LabelRef("sealSSETail"))
}

// ----------------------------------------------------------------------------
// Special optimization for buffers smaller than 321 bytes

// For up to 320 bytes of ciphertext and 64 bytes for the poly key, we process six blocks
func seal320AVX2() {
	Label("seal320AVX2")
	VMOVDQA(AA0, AA1)
	VMOVDQA(BB0, BB1)
	VMOVDQA(CC0, CC1)
	avx2IncMask := avx2IncMask_DATA()
	VPADDD(avx2IncMask, DD0, DD1)
	VMOVDQA(AA0, AA2)
	VMOVDQA(BB0, BB2)
	VMOVDQA(CC0, CC2)
	VPADDD(avx2IncMask, DD1, DD2)
	VMOVDQA(BB0, TT1)
	VMOVDQA(CC0, TT2)
	VMOVDQA(DD0, TT3)
	MOVQ(U32(10), itr2)
}

func sealAVX2320InnerCipherLoop() {
	Label("sealAVX2320InnerCipherLoop")
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
	VPALIGNR(Imm(4), BB0, BB0, BB0)
	VPALIGNR(Imm(4), BB1, BB1, BB1)
	VPALIGNR(Imm(4), BB2, BB2, BB2)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(12), DD0, DD0, DD0)
	VPALIGNR(Imm(12), DD1, DD1, DD1)
	VPALIGNR(Imm(12), DD2, DD2, DD2)
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
	VPALIGNR(Imm(12), BB0, BB0, BB0)
	VPALIGNR(Imm(12), BB1, BB1, BB1)
	VPALIGNR(Imm(12), BB2, BB2, BB2)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(4), DD0, DD0, DD0)
	VPALIGNR(Imm(4), DD1, DD1, DD1)
	VPALIGNR(Imm(4), DD2, DD2, DD2)
	DECQ(itr2)
	JNE(LabelRef("sealAVX2320InnerCipherLoop"))

	chacha20Constants := chacha20Constants_DATA()
	VMOVDQA(chacha20Constants, TT0)
	VPADDD(TT0, AA0, AA0)
	VPADDD(TT0, AA1, AA1)
	VPADDD(TT0, AA2, AA2)
	VPADDD(TT1, BB0, BB0)
	VPADDD(TT1, BB1, BB1)
	VPADDD(TT1, BB2, BB2)
	VPADDD(TT2, CC0, CC0)
	VPADDD(TT2, CC1, CC1)
	VPADDD(TT2, CC2, CC2)
	avx2IncMask := avx2IncMask_DATA()
	VMOVDQA(avx2IncMask, TT0)
	VPADDD(TT3, DD0, DD0)
	VPADDD(TT0, TT3, TT3)
	VPADDD(TT3, DD1, DD1)
	VPADDD(TT0, TT3, TT3)
	VPADDD(TT3, DD2, DD2)

	Comment("Clamp and store poly key")
	VPERM2I128(Imm(0x02), AA0, BB0, TT0)
	polyClampMask := polyClampMask_DATA()
	VPAND(polyClampMask, TT0, TT0)
	VMOVDQA(TT0, rsStoreAVX2)

	Comment("Stream for up to 320 bytes")
	VPERM2I128(Imm(0x13), AA0, BB0, AA0)
	VPERM2I128(Imm(0x13), CC0, DD0, BB0)
	VPERM2I128(Imm(0x02), AA1, BB1, CC0)
	VPERM2I128(Imm(0x02), CC1, DD1, DD0)
	VPERM2I128(Imm(0x13), AA1, BB1, AA1)
	VPERM2I128(Imm(0x13), CC1, DD1, BB1)
	VPERM2I128(Imm(0x02), AA2, BB2, CC1)
	VPERM2I128(Imm(0x02), CC2, DD2, DD1)
	VPERM2I128(Imm(0x13), AA2, BB2, AA2)
	VPERM2I128(Imm(0x13), CC2, DD2, BB2)
	JMP(LabelRef("sealAVX2ShortSeal"))
}

// Need to decrypt up to 128 bytes - prepare two blocks:
//   - If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed.
//   - If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed.
func sealAVX2Tail128() {
	Label("sealAVX2Tail128")
	chacha20Constants := chacha20Constants_DATA()
	VMOVDQA(chacha20Constants, AA0)
	VMOVDQA(state1StoreAVX2, BB0)
	VMOVDQA(state2StoreAVX2, CC0)
	VMOVDQA(ctr3StoreAVX2, DD0)
	avx2IncMask := avx2IncMask_DATA()
	VPADDD(avx2IncMask, DD0, DD0)
	VMOVDQA(DD0, DD1)
}

func sealAVX2Tail128LoopA() {
	Label("sealAVX2Tail128LoopA")
	polyAdd(Mem{Base: oup}.Offset(0))
	polyMul()
	LEAQ(Mem{Base: oup}.Offset(16), oup)
}

func sealAVX2Tail128LoopB() {
	Label("sealAVX2Tail128LoopB")
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	polyAdd(Mem{Base: oup}.Offset(0))
	polyMul()
	VPALIGNR(Imm(4), BB0, BB0, BB0)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(12), DD0, DD0, DD0)
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	polyAdd(Mem{Base: oup}.Offset(16))
	polyMul()
	LEAQ(Mem{Base: oup}.Offset(32), oup)
	VPALIGNR(Imm(12), BB0, BB0, BB0)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(4), DD0, DD0, DD0)
	DECQ(itr1)
	JG(LabelRef("sealAVX2Tail128LoopA"))
	DECQ(itr2)
	JGE(LabelRef("sealAVX2Tail128LoopB"))

	chacha20Constants := chacha20Constants_DATA()
	VPADDD(chacha20Constants, AA0, AA1)
	VPADDD(state1StoreAVX2, BB0, BB1)
	VPADDD(state2StoreAVX2, CC0, CC1)
	VPADDD(DD1, DD0, DD1)

	VPERM2I128(Imm(0x02), AA1, BB1, AA0)
	VPERM2I128(Imm(0x02), CC1, DD1, BB0)
	VPERM2I128(Imm(0x13), AA1, BB1, CC0)
	VPERM2I128(Imm(0x13), CC1, DD1, DD0)
	JMP(LabelRef("sealAVX2ShortSealLoop"))
}

// ----------------------------------------------------------------------------
// Special optimization for the last 256 bytes of ciphertext

// Need to decrypt up to 256 bytes - prepare two blocks
//   - If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed
//   - If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed
func sealAVX2Tail256() {
	Label("sealAVX2Tail256")
	chacha20Constants := chacha20Constants_DATA()
	VMOVDQA(chacha20Constants, AA0)
	VMOVDQA(chacha20Constants, AA1)
	VMOVDQA(state1StoreAVX2, BB0)
	VMOVDQA(state1StoreAVX2, BB1)
	VMOVDQA(state2StoreAVX2, CC0)
	VMOVDQA(state2StoreAVX2, CC1)
	VMOVDQA(ctr3StoreAVX2, DD0)
	avx2IncMask := avx2IncMask_DATA()
	VPADDD(avx2IncMask, DD0, DD0)
	VPADDD(avx2IncMask, DD0, DD1)
	VMOVDQA(DD0, TT1)
	VMOVDQA(DD1, TT2)
}

func sealAVX2Tail256LoopA() {
	Label("sealAVX2Tail256LoopA")
	polyAdd(Mem{Base: oup}.Offset(0))
	polyMul()
	LEAQ(Mem{Base: oup}.Offset(16), oup)
}

// LIne 2493
func sealAVX2Tail256LoopB() {
	Label("sealAVX2Tail256LoopB")
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	polyAdd(Mem{Base: oup}.Offset(0))
	polyMul()
	VPALIGNR(Imm(4), BB0, BB0, BB0)
	VPALIGNR(Imm(4), BB1, BB1, BB1)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(12), DD0, DD0, DD0)
	VPALIGNR(Imm(12), DD1, DD1, DD1)
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	polyAdd(Mem{Base: oup}.Offset(16))
	polyMul()
	LEAQ(Mem{Base: oup}.Offset(32), oup)
	VPALIGNR(Imm(12), BB0, BB0, BB0)
	VPALIGNR(Imm(12), BB1, BB1, BB1)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(4), DD0, DD0, DD0)
	VPALIGNR(Imm(4), DD1, DD1, DD1)
	DECQ(itr1)
	JG(LabelRef("sealAVX2Tail256LoopA"))
	DECQ(itr2)
	JGE(LabelRef("sealAVX2Tail256LoopB"))

	chacha20Constants := chacha20Constants_DATA()
	VPADDD(chacha20Constants, AA0, AA0)
	VPADDD(chacha20Constants, AA1, AA1)
	VPADDD(state1StoreAVX2, BB0, BB0)
	VPADDD(state1StoreAVX2, BB1, BB1)
	VPADDD(state2StoreAVX2, CC0, CC0)
	VPADDD(state2StoreAVX2, CC1, CC1)
	VPADDD(TT1, DD0, DD0)
	VPADDD(TT2, DD1, DD1)
	VPERM2I128(Imm(0x02), AA0, BB0, TT0)
	VPERM2I128(Imm(0x02), CC0, DD0, TT1)
	VPERM2I128(Imm(0x13), AA0, BB0, TT2)
	VPERM2I128(Imm(0x13), CC0, DD0, TT3)
	VPXOR(Mem{Base: inp}.Offset(0*32), TT0, TT0)
	VPXOR(Mem{Base: inp}.Offset(1*32), TT1, TT1)
	VPXOR(Mem{Base: inp}.Offset(2*32), TT2, TT2)
	VPXOR(Mem{Base: inp}.Offset(3*32), TT3, TT3)
	VMOVDQU(TT0, Mem{Base: oup}.Offset(0*32))
	VMOVDQU(TT1, Mem{Base: oup}.Offset(1*32))
	VMOVDQU(TT2, Mem{Base: oup}.Offset(2*32))
	VMOVDQU(TT3, Mem{Base: oup}.Offset(3*32))
	MOVQ(U32(128), itr1)
	LEAQ(Mem{Base: inp}.Offset(128), inp)
	SUBQ(Imm(128), inl)
	VPERM2I128(Imm(0x02), AA1, BB1, AA0)
	VPERM2I128(Imm(0x02), CC1, DD1, BB0)
	VPERM2I128(Imm(0x13), AA1, BB1, CC0)
	VPERM2I128(Imm(0x13), CC1, DD1, DD0)

	JMP(LabelRef("sealAVX2SealHash"))
}

// ----------------------------------------------------------------------------
// Special optimization for the last 384 bytes of ciphertext

// Need to decrypt up to 384 bytes - prepare two blocks
//   - If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed
//   - If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed
func sealAVX2Tail384() {
	Label("sealAVX2Tail384")
	chacha20Constants := chacha20Constants_DATA()
	VMOVDQA(chacha20Constants, AA0)
	VMOVDQA(AA0, AA1)
	VMOVDQA(AA0, AA2)
	VMOVDQA(state1StoreAVX2, BB0)
	VMOVDQA(BB0, BB1)
	VMOVDQA(BB0, BB2)
	VMOVDQA(state2StoreAVX2, CC0)
	VMOVDQA(CC0, CC1)
	VMOVDQA(CC0, CC2)
	VMOVDQA(ctr3StoreAVX2, DD0)
	avx2IncMask := avx2IncMask_DATA()
	VPADDD(avx2IncMask, DD0, DD0)
	VPADDD(avx2IncMask, DD0, DD1)
	VPADDD(avx2IncMask, DD1, DD2)
	VMOVDQA(DD0, TT1)
	VMOVDQA(DD1, TT2)
	VMOVDQA(DD2, TT3)
}

func sealAVX2Tail384LoopA() {
	Label("sealAVX2Tail384LoopA")
	polyAdd(Mem{Base: oup}.Offset(0))
	polyMul()
	LEAQ(Mem{Base: oup}.Offset(16), oup)
}

func sealAVX2Tail384LoopB() {
	Label("sealAVX2Tail384LoopB")
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
	polyAdd(Mem{Base: oup}.Offset(0))
	polyMul()
	VPALIGNR(Imm(4), BB0, BB0, BB0)
	VPALIGNR(Imm(4), BB1, BB1, BB1)
	VPALIGNR(Imm(4), BB2, BB2, BB2)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(12), DD0, DD0, DD0)
	VPALIGNR(Imm(12), DD1, DD1, DD1)
	VPALIGNR(Imm(12), DD2, DD2, DD2)
	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
	chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
	polyAdd(Mem{Base: oup}.Offset(16))
	polyMul()
	LEAQ(Mem{Base: oup}.Offset(32), oup)
	VPALIGNR(Imm(12), BB0, BB0, BB0)
	VPALIGNR(Imm(12), BB1, BB1, BB1)
	VPALIGNR(Imm(12), BB2, BB2, BB2)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(4), DD0, DD0, DD0)
	VPALIGNR(Imm(4), DD1, DD1, DD1)
	VPALIGNR(Imm(4), DD2, DD2, DD2)
	DECQ(itr1)
	JG(LabelRef("sealAVX2Tail384LoopA"))
	DECQ(itr2)
	JGE(LabelRef("sealAVX2Tail384LoopB"))

	chacha20Constants := chacha20Constants_DATA()
	VPADDD(chacha20Constants, AA0, AA0)
	VPADDD(chacha20Constants, AA1, AA1)
	VPADDD(chacha20Constants, AA2, AA2)
	VPADDD(state1StoreAVX2, BB0, BB0)
	VPADDD(state1StoreAVX2, BB1, BB1)
	VPADDD(state1StoreAVX2, BB2, BB2)
	VPADDD(state2StoreAVX2, CC0, CC0)
	VPADDD(state2StoreAVX2, CC1, CC1)
	VPADDD(state2StoreAVX2, CC2, CC2)
	VPADDD(TT1, DD0, DD0)
	VPADDD(TT2, DD1, DD1)
	VPADDD(TT3, DD2, DD2)
	VPERM2I128(Imm(0x02), AA0, BB0, TT0)
	VPERM2I128(Imm(0x02), CC0, DD0, TT1)
	VPERM2I128(Imm(0x13), AA0, BB0, TT2)
	VPERM2I128(Imm(0x13), CC0, DD0, TT3)
	VPXOR(Mem{Base: inp}.Offset(0*32), TT0, TT0)
	VPXOR(Mem{Base: inp}.Offset(1*32), TT1, TT1)
	VPXOR(Mem{Base: inp}.Offset(2*32), TT2, TT2)
	VPXOR(Mem{Base: inp}.Offset(3*32), TT3, TT3)
	VMOVDQU(TT0, Mem{Base: oup}.Offset(0*32))
	VMOVDQU(TT1, Mem{Base: oup}.Offset(1*32))
	VMOVDQU(TT2, Mem{Base: oup}.Offset(2*32))
	VMOVDQU(TT3, Mem{Base: oup}.Offset(3*32))
	VPERM2I128(Imm(0x02), AA1, BB1, TT0)
	VPERM2I128(Imm(0x02), CC1, DD1, TT1)
	VPERM2I128(Imm(0x13), AA1, BB1, TT2)
	VPERM2I128(Imm(0x13), CC1, DD1, TT3)
	VPXOR(Mem{Base: inp}.Offset(4*32), TT0, TT0)
	VPXOR(Mem{Base: inp}.Offset(5*32), TT1, TT1)
	VPXOR(Mem{Base: inp}.Offset(6*32), TT2, TT2)
	VPXOR(Mem{Base: inp}.Offset(7*32), TT3, TT3)
	VMOVDQU(TT0, Mem{Base: oup}.Offset(4*32))
	VMOVDQU(TT1, Mem{Base: oup}.Offset(5*32))
	VMOVDQU(TT2, Mem{Base: oup}.Offset(6*32))
	VMOVDQU(TT3, Mem{Base: oup}.Offset(7*32))
	MOVQ(U32(256), itr1)
	LEAQ(Mem{Base: inp}.Offset(256), inp)
	SUBQ(U32(256), inl)
	VPERM2I128(Imm(0x02), AA2, BB2, AA0)
	VPERM2I128(Imm(0x02), CC2, DD2, BB0)
	VPERM2I128(Imm(0x13), AA2, BB2, CC0)
	VPERM2I128(Imm(0x13), CC2, DD2, DD0)

	JMP(LabelRef("sealAVX2SealHash"))
}

// ----------------------------------------------------------------------------
// Special optimization for the last 512 bytes of ciphertext

// Need to decrypt up to 512 bytes - prepare two blocks
//   - If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed
//   - If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed
func sealAVX2Tail512() {
	Label("sealAVX2Tail512")
	chacha20Constants := chacha20Constants_DATA()
	VMOVDQA(chacha20Constants, AA0)
	VMOVDQA(AA0, AA1)
	VMOVDQA(AA0, AA2)
	VMOVDQA(AA0, AA3)
	VMOVDQA(state1StoreAVX2, BB0)
	VMOVDQA(BB0, BB1)
	VMOVDQA(BB0, BB2)
	VMOVDQA(BB0, BB3)
	VMOVDQA(state2StoreAVX2, CC0)
	VMOVDQA(CC0, CC1)
	VMOVDQA(CC0, CC2)
	VMOVDQA(CC0, CC3)
	VMOVDQA(ctr3StoreAVX2, DD0)
	avx2IncMask := avx2IncMask_DATA()
	VPADDD(avx2IncMask, DD0, DD0)
	VPADDD(avx2IncMask, DD0, DD1)
	VPADDD(avx2IncMask, DD1, DD2)
	VPADDD(avx2IncMask, DD2, DD3)
	VMOVDQA(DD0, ctr0StoreAVX2)
	VMOVDQA(DD1, ctr1StoreAVX2)
	VMOVDQA(DD2, ctr2StoreAVX2)
	VMOVDQA(DD3, ctr3StoreAVX2)
}

func sealAVX2Tail512LoopA() {
	Label("sealAVX2Tail512LoopA")
	polyAdd(Mem{Base: oup}.Offset(0))
	polyMul()
	LEAQ(Mem{Base: oup}.Offset(16), oup)
}

func sealAVX2Tail512LoopB() {
	Label("sealAVX2Tail512LoopB")
	VPADDD(BB0, AA0, AA0)
	VPADDD(BB1, AA1, AA1)
	VPADDD(BB2, AA2, AA2)
	VPADDD(BB3, AA3, AA3)
	VPXOR(AA0, DD0, DD0)
	VPXOR(AA1, DD1, DD1)
	VPXOR(AA2, DD2, DD2)
	VPXOR(AA3, DD3, DD3)
	rol16 := rol16_DATA()
	VPSHUFB(rol16, DD0, DD0)
	VPSHUFB(rol16, DD1, DD1)
	VPSHUFB(rol16, DD2, DD2)
	VPSHUFB(rol16, DD3, DD3)
	VPADDD(DD0, CC0, CC0)
	VPADDD(DD1, CC1, CC1)
	VPADDD(DD2, CC2, CC2)
	VPADDD(DD3, CC3, CC3)
	VPXOR(CC0, BB0, BB0)
	VPXOR(CC1, BB1, BB1)
	VPXOR(CC2, BB2, BB2)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(CC3, tmpStoreAVX2)
	VPSLLD(Imm(12), BB0, CC3)
	VPSRLD(Imm(20), BB0, BB0)
	VPXOR(CC3, BB0, BB0)
	VPSLLD(Imm(12), BB1, CC3)
	VPSRLD(Imm(20), BB1, BB1)
	VPXOR(CC3, BB1, BB1)
	VPSLLD(Imm(12), BB2, CC3)
	VPSRLD(Imm(20), BB2, BB2)
	VPXOR(CC3, BB2, BB2)
	VPSLLD(Imm(12), BB3, CC3)
	VPSRLD(Imm(20), BB3, BB3)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(tmpStoreAVX2, CC3)
	polyAdd(Mem{Base: oup}.Offset(0 * 8))
	polyMulAVX2()
	VPADDD(BB0, AA0, AA0)
	VPADDD(BB1, AA1, AA1)
	VPADDD(BB2, AA2, AA2)
	VPADDD(BB3, AA3, AA3)
	VPXOR(AA0, DD0, DD0)
	VPXOR(AA1, DD1, DD1)
	VPXOR(AA2, DD2, DD2)
	VPXOR(AA3, DD3, DD3)
	rol8 := rol8_DATA()
	VPSHUFB(rol8, DD0, DD0)
	VPSHUFB(rol8, DD1, DD1)
	VPSHUFB(rol8, DD2, DD2)
	VPSHUFB(rol8, DD3, DD3)
	VPADDD(DD0, CC0, CC0)
	VPADDD(DD1, CC1, CC1)
	VPADDD(DD2, CC2, CC2)
	VPADDD(DD3, CC3, CC3)
	VPXOR(CC0, BB0, BB0)
	VPXOR(CC1, BB1, BB1)
	VPXOR(CC2, BB2, BB2)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(CC3, tmpStoreAVX2)
	VPSLLD(Imm(7), BB0, CC3)
	VPSRLD(Imm(25), BB0, BB0)
	VPXOR(CC3, BB0, BB0)
	VPSLLD(Imm(7), BB1, CC3)
	VPSRLD(Imm(25), BB1, BB1)
	VPXOR(CC3, BB1, BB1)
	VPSLLD(Imm(7), BB2, CC3)
	VPSRLD(Imm(25), BB2, BB2)
	VPXOR(CC3, BB2, BB2)
	VPSLLD(Imm(7), BB3, CC3)
	VPSRLD(Imm(25), BB3, BB3)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(tmpStoreAVX2, CC3)
	VPALIGNR(Imm(4), BB0, BB0, BB0)
	VPALIGNR(Imm(4), BB1, BB1, BB1)
	VPALIGNR(Imm(4), BB2, BB2, BB2)
	VPALIGNR(Imm(4), BB3, BB3, BB3)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(8), CC3, CC3, CC3)
	VPALIGNR(Imm(12), DD0, DD0, DD0)
	VPALIGNR(Imm(12), DD1, DD1, DD1)
	VPALIGNR(Imm(12), DD2, DD2, DD2)
	VPALIGNR(Imm(12), DD3, DD3, DD3)
	VPADDD(BB0, AA0, AA0)
	VPADDD(BB1, AA1, AA1)
	VPADDD(BB2, AA2, AA2)
	VPADDD(BB3, AA3, AA3)
	VPXOR(AA0, DD0, DD0)
	VPXOR(AA1, DD1, DD1)
	VPXOR(AA2, DD2, DD2)
	VPXOR(AA3, DD3, DD3)
	VPSHUFB(rol16, DD0, DD0)
	VPSHUFB(rol16, DD1, DD1)
	VPSHUFB(rol16, DD2, DD2)
	VPSHUFB(rol16, DD3, DD3)
	VPADDD(DD0, CC0, CC0)
	VPADDD(DD1, CC1, CC1)
	VPADDD(DD2, CC2, CC2)
	VPADDD(DD3, CC3, CC3)
	VPXOR(CC0, BB0, BB0)
	VPXOR(CC1, BB1, BB1)
	VPXOR(CC2, BB2, BB2)
	VPXOR(CC3, BB3, BB3)
	polyAdd(Mem{Base: oup}.Offset(2 * 8))
	polyMulAVX2()
	LEAQ(Mem{Base: oup}.Offset(4*8), oup)
	VMOVDQA(CC3, tmpStoreAVX2)
	VPSLLD(Imm(12), BB0, CC3)
	VPSRLD(Imm(20), BB0, BB0)
	VPXOR(CC3, BB0, BB0)
	VPSLLD(Imm(12), BB1, CC3)
	VPSRLD(Imm(20), BB1, BB1)
	VPXOR(CC3, BB1, BB1)
	VPSLLD(Imm(12), BB2, CC3)
	VPSRLD(Imm(20), BB2, BB2)
	VPXOR(CC3, BB2, BB2)
	VPSLLD(Imm(12), BB3, CC3)
	VPSRLD(Imm(20), BB3, BB3)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(tmpStoreAVX2, CC3)
	VPADDD(BB0, AA0, AA0)
	VPADDD(BB1, AA1, AA1)
	VPADDD(BB2, AA2, AA2)
	VPADDD(BB3, AA3, AA3)
	VPXOR(AA0, DD0, DD0)
	VPXOR(AA1, DD1, DD1)
	VPXOR(AA2, DD2, DD2)
	VPXOR(AA3, DD3, DD3)
	VPSHUFB(rol8, DD0, DD0)
	VPSHUFB(rol8, DD1, DD1)
	VPSHUFB(rol8, DD2, DD2)
	VPSHUFB(rol8, DD3, DD3)
	VPADDD(DD0, CC0, CC0)
	VPADDD(DD1, CC1, CC1)
	VPADDD(DD2, CC2, CC2)
	VPADDD(DD3, CC3, CC3)
	VPXOR(CC0, BB0, BB0)
	VPXOR(CC1, BB1, BB1)
	VPXOR(CC2, BB2, BB2)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(CC3, tmpStoreAVX2)
	VPSLLD(Imm(7), BB0, CC3)
	VPSRLD(Imm(25), BB0, BB0)
	VPXOR(CC3, BB0, BB0)
	VPSLLD(Imm(7), BB1, CC3)
	VPSRLD(Imm(25), BB1, BB1)
	VPXOR(CC3, BB1, BB1)
	VPSLLD(Imm(7), BB2, CC3)
	VPSRLD(Imm(25), BB2, BB2)
	VPXOR(CC3, BB2, BB2)
	VPSLLD(Imm(7), BB3, CC3)
	VPSRLD(Imm(25), BB3, BB3)
	VPXOR(CC3, BB3, BB3)
	VMOVDQA(tmpStoreAVX2, CC3)
	VPALIGNR(Imm(12), BB0, BB0, BB0)
	VPALIGNR(Imm(12), BB1, BB1, BB1)
	VPALIGNR(Imm(12), BB2, BB2, BB2)
	VPALIGNR(Imm(12), BB3, BB3, BB3)
	VPALIGNR(Imm(8), CC0, CC0, CC0)
	VPALIGNR(Imm(8), CC1, CC1, CC1)
	VPALIGNR(Imm(8), CC2, CC2, CC2)
	VPALIGNR(Imm(8), CC3, CC3, CC3)
	VPALIGNR(Imm(4), DD0, DD0, DD0)
	VPALIGNR(Imm(4), DD1, DD1, DD1)
	VPALIGNR(Imm(4), DD2, DD2, DD2)
	VPALIGNR(Imm(4), DD3, DD3, DD3)

	DECQ(itr1)
	JG(LabelRef("sealAVX2Tail512LoopA"))
	DECQ(itr2)
	JGE(LabelRef("sealAVX2Tail512LoopB"))

	chacha20Constants := chacha20Constants_DATA()
	VPADDD(chacha20Constants, AA0, AA0)
	VPADDD(chacha20Constants, AA1, AA1)
	VPADDD(chacha20Constants, AA2, AA2)
	VPADDD(chacha20Constants, AA3, AA3)
	VPADDD(state1StoreAVX2, BB0, BB0)
	VPADDD(state1StoreAVX2, BB1, BB1)
	VPADDD(state1StoreAVX2, BB2, BB2)
	VPADDD(state1StoreAVX2, BB3, BB3)
	VPADDD(state2StoreAVX2, CC0, CC0)
	VPADDD(state2StoreAVX2, CC1, CC1)
	VPADDD(state2StoreAVX2, CC2, CC2)
	VPADDD(state2StoreAVX2, CC3, CC3)
	VPADDD(ctr0StoreAVX2, DD0, DD0)
	VPADDD(ctr1StoreAVX2, DD1, DD1)
	VPADDD(ctr2StoreAVX2, DD2, DD2)
	VPADDD(ctr3StoreAVX2, DD3, DD3)
	VMOVDQA(CC3, tmpStoreAVX2)
	VPERM2I128(Imm(0x02), AA0, BB0, CC3)
	VPXOR(Mem{Base: inp}.Offset(0*32), CC3, CC3)
	VMOVDQU(CC3, Mem{Base: oup}.Offset(0*32))
	VPERM2I128(Imm(0x02), CC0, DD0, CC3)
	VPXOR(Mem{Base: inp}.Offset(1*32), CC3, CC3)
	VMOVDQU(CC3, Mem{Base: oup}.Offset(1*32))
	VPERM2I128(Imm(0x13), AA0, BB0, CC3)
	VPXOR(Mem{Base: inp}.Offset(2*32), CC3, CC3)
	VMOVDQU(CC3, Mem{Base: oup}.Offset(2*32))
	VPERM2I128(Imm(0x13), CC0, DD0, CC3)
	VPXOR(Mem{Base: inp}.Offset(3*32), CC3, CC3)
	VMOVDQU(CC3, Mem{Base: oup}.Offset(3*32))

	VPERM2I128(Imm(0x02), AA1, BB1, AA0)
	VPERM2I128(Imm(0x02), CC1, DD1, BB0)
	VPERM2I128(Imm(0x13), AA1, BB1, CC0)
	VPERM2I128(Imm(0x13), CC1, DD1, DD0)
	VPXOR(Mem{Base: inp}.Offset(4*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(5*32), BB0, BB0)
	VPXOR(Mem{Base: inp}.Offset(6*32), CC0, CC0)
	VPXOR(Mem{Base: inp}.Offset(7*32), DD0, DD0)
	VMOVDQU(AA0, Mem{Base: oup}.Offset(4*32))
	VMOVDQU(BB0, Mem{Base: oup}.Offset(5*32))
	VMOVDQU(CC0, Mem{Base: oup}.Offset(6*32))
	VMOVDQU(DD0, Mem{Base: oup}.Offset(7*32))

	VPERM2I128(Imm(0x02), AA2, BB2, AA0)
	VPERM2I128(Imm(0x02), CC2, DD2, BB0)
	VPERM2I128(Imm(0x13), AA2, BB2, CC0)
	VPERM2I128(Imm(0x13), CC2, DD2, DD0)
	VPXOR(Mem{Base: inp}.Offset(8*32), AA0, AA0)
	VPXOR(Mem{Base: inp}.Offset(9*32), BB0, BB0)
	VPXOR(Mem{Base: inp}.Offset(10*32), CC0, CC0)
	VPXOR(Mem{Base: inp}.Offset(11*32), DD0, DD0)
	VMOVDQU(AA0, Mem{Base: oup}.Offset(8*32))
	VMOVDQU(BB0, Mem{Base: oup}.Offset(9*32))
	VMOVDQU(CC0, Mem{Base: oup}.Offset(10*32))
	VMOVDQU(DD0, Mem{Base: oup}.Offset(11*32))

	MOVQ(U32(384), itr1)
	LEAQ(Mem{Base: inp}.Offset(384), inp)
	SUBQ(U32(384), inl)
	VPERM2I128(Imm(0x02), AA3, BB3, AA0)
	VPERM2I128(Imm(0x02), tmpStoreAVX2, DD3, BB0)
	VPERM2I128(Imm(0x13), AA3, BB3, CC0)
	VPERM2I128(Imm(0x13), tmpStoreAVX2, DD3, DD0)

	JMP(LabelRef("sealAVX2SealHash"))
}

// ##~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~DATA SECTION~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~##

var (
	// Pointers for memoizing DATA section symbols
	chacha20Constants_DATA_ptr,
	rol16_DATA_ptr,
	rol8_DATA_ptr,
	sseIncMask_DATA_ptr,
	avx2IncMask_DATA_ptr,
	avx2InitMask_DATA_ptr,
	polyClampMask_DATA_ptr,
	andMask_DATA_ptr *Mem
)

var nothingUpMySleeve = [8]uint32{
	0x61707865,
	0x3320646e,
	0x79622d32,
	0x6b206574,
	0x61707865,
	0x3320646e,
	0x79622d32,
	0x6b206574,
}

// ChaCha20 constants
func chacha20Constants_DATA() Mem {
	if chacha20Constants_DATA_ptr != nil {
		return *chacha20Constants_DATA_ptr
	}

	chacha20Constants := GLOBL(ThatPeskyUnicodeDot+"chacha20Constants", NOPTR|RODATA)
	chacha20Constants_DATA_ptr = &chacha20Constants
	for i, v := range nothingUpMySleeve {
		DATA(i*4, U32(v))
	}
	return chacha20Constants
}

var rol16Consts = [4]uint64{
	0x0504070601000302,
	0x0D0C0F0E09080B0A,
	0x0504070601000302,
	0x0D0C0F0E09080B0A,
}

// <<< 16 with PSHUFB
func rol16_DATA() Mem {
	if rol16_DATA_ptr != nil {
		return *rol16_DATA_ptr
	}

	rol16 := GLOBL(ThatPeskyUnicodeDot+"rol16", NOPTR|RODATA)
	rol16_DATA_ptr = &rol16
	for i, v := range rol16Consts {
		DATA(i*8, U64(v))
	}
	return rol16
}

var rol8Consts = [4]uint64{
	0x0605040702010003,
	0x0E0D0C0F0A09080B,
	0x0605040702010003,
	0x0E0D0C0F0A09080B,
}

// <<< 8 with PSHUFB
func rol8_DATA() Mem {
	if rol8_DATA_ptr != nil {
		return *rol8_DATA_ptr
	}

	rol8 := GLOBL(ThatPeskyUnicodeDot+"rol8", NOPTR|RODATA)
	rol8_DATA_ptr = &rol8
	for i, v := range rol8Consts {
		DATA(i*8, U64(v))
	}
	return rol8
}

var avx2InitMaskConsts = [4]uint64{
	0x0,
	0x0,
	0x1,
	0x0,
}

func avx2InitMask_DATA() Mem {
	if avx2InitMask_DATA_ptr != nil {
		return *avx2InitMask_DATA_ptr
	}

	avx2InitMask := GLOBL(ThatPeskyUnicodeDot+"avx2InitMask", NOPTR|RODATA)
	avx2InitMask_DATA_ptr = &avx2InitMask
	for i, v := range avx2InitMaskConsts {
		DATA(i*8, U64(v))
	}
	return avx2InitMask
}

var avx2IncMaskConsts = [4]uint64{
	0x2,
	0x0,
	0x2,
	0x0,
}

func avx2IncMask_DATA() Mem {
	if avx2IncMask_DATA_ptr != nil {
		return *avx2IncMask_DATA_ptr
	}

	avx2IncMask := GLOBL(ThatPeskyUnicodeDot+"avx2IncMask", NOPTR|RODATA)
	avx2IncMask_DATA_ptr = &avx2IncMask
	for i, v := range avx2IncMaskConsts {
		DATA(i*8, U64(v))
	}
	return avx2IncMask
}

var polyClampMaskConsts = [4]uint64{
	0x0FFFFFFC0FFFFFFF,
	0x0FFFFFFC0FFFFFFC,
	0xFFFFFFFFFFFFFFFF,
	0xFFFFFFFFFFFFFFFF,
}

// Poly1305 key clamp
func polyClampMask_DATA() Mem {
	if polyClampMask_DATA_ptr != nil {
		return *polyClampMask_DATA_ptr
	}

	polyClampMask := GLOBL(ThatPeskyUnicodeDot+"polyClampMask", NOPTR|RODATA)
	polyClampMask_DATA_ptr = &polyClampMask
	for i, v := range polyClampMaskConsts {
		DATA(i*8, U64(v))
	}
	return polyClampMask
}

var sseIncMaskConsts = [2]uint64{
	0x1,
	0x0,
}

func sseIncMask_DATA() Mem {
	if sseIncMask_DATA_ptr != nil {
		return *sseIncMask_DATA_ptr
	}

	sseIncMask := GLOBL(ThatPeskyUnicodeDot+"sseIncMask", NOPTR|RODATA)
	sseIncMask_DATA_ptr = &sseIncMask
	for i, v := range sseIncMaskConsts {
		DATA(i*8, U64(v))
	}
	return sseIncMask
}

var andMaskConsts = [30]uint64{
	0x00000000000000ff,
	0x0000000000000000,
	0x000000000000ffff,
	0x0000000000000000,
	0x0000000000ffffff,
	0x0000000000000000,
	0x00000000ffffffff,
	0x0000000000000000,
	0x000000ffffffffff,
	0x0000000000000000,
	0x0000ffffffffffff,
	0x0000000000000000,
	0x00ffffffffffffff,
	0x0000000000000000,
	0xffffffffffffffff,
	0x0000000000000000,
	0xffffffffffffffff,
	0x00000000000000ff,
	0xffffffffffffffff,
	0x000000000000ffff,
	0xffffffffffffffff,
	0x0000000000ffffff,
	0xffffffffffffffff,
	0x00000000ffffffff,
	0xffffffffffffffff,
	0x000000ffffffffff,
	0xffffffffffffffff,
	0x0000ffffffffffff,
	0xffffffffffffffff,
	0x00ffffffffffffff,
}

func andMask_DATA() Mem {
	if andMask_DATA_ptr != nil {
		return *andMask_DATA_ptr
	}

	andMask := GLOBL(ThatPeskyUnicodeDot+"andMask", NOPTR|RODATA)
	andMask_DATA_ptr = &andMask
	for i, v := range andMaskConsts {
		DATA(i*8, U64(v))
	}
	return andMask
}

// removePeskyUnicodeDot strips the dot from the relevant TEXT directives such that they
// can exist as internal assembly functions
//
// Avo v0.6.0 does not support the generation of internal assembly functions. Go's unicode
// dot tells the compiler to link a TEXT symbol to a function in the current Go package
// (or another package if specified). Avo unconditionally prepends the unicode dot to all
// TEXT symbols, making it impossible to emit an internal function without this hack.
//
// There is a pending PR to add internal functions to Avo:
// https://github.com/mmcloughlin/avo/pull/443
//
// If merged it should allow the usage of InternalFunction("NAME") for the specified functions
func removePeskyUnicodeDot(internalFunctions []string, target string) {
	bytes, err := os.ReadFile(target)
	if err != nil {
		panic(err)
	}

	content := string(bytes)

	for _, from := range internalFunctions {
		to := strings.ReplaceAll(from, ThatPeskyUnicodeDot, "")
		content = strings.ReplaceAll(content, from, to)
	}

	err = os.WriteFile(target, []byte(content), 0644)
	if err != nil {
		panic(err)
	}
}