// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build !purego

#include "textflag.h"

#define B0 V0
#define B1 V1
#define B2 V2
#define B3 V3
#define B4 V4
#define B5 V5
#define B6 V6
#define B7 V7

#define ACC0 V8
#define ACC1 V9
#define ACCM V10

#define T0 V11
#define T1 V12
#define T2 V13
#define T3 V14

#define POLY V15
#define ZERO V16
#define INC V17
#define CTR V18

#define K0 V19
#define K1 V20
#define K2 V21
#define K3 V22
#define K4 V23
#define K5 V24
#define K6 V25
#define K7 V26
#define K8 V27
#define K9 V28
#define K10 V29
#define K11 V30
#define KLAST V31

#define reduce() \
	VEOR	ACC0.B16, ACCM.B16, ACCM.B16     \
	VEOR	ACC1.B16, ACCM.B16, ACCM.B16     \
	VEXT	$8, ZERO.B16, ACCM.B16, T0.B16   \
	VEXT	$8, ACCM.B16, ZERO.B16, ACCM.B16 \
	VEOR	ACCM.B16, ACC0.B16, ACC0.B16     \
	VEOR	T0.B16, ACC1.B16, ACC1.B16       \
	VPMULL	POLY.D1, ACC0.D1, T0.Q1          \
	VEXT	$8, ACC0.B16, ACC0.B16, ACC0.B16 \
	VEOR	T0.B16, ACC0.B16, ACC0.B16       \
	VPMULL	POLY.D1, ACC0.D1, T0.Q1          \
	VEOR	T0.B16, ACC1.B16, ACC1.B16       \
	VEXT	$8, ACC1.B16, ACC1.B16, ACC1.B16 \
	VEOR	ACC1.B16, ACC0.B16, ACC0.B16     \

// func gcmAesFinish(productTable *[256]byte, tagMask, T *[16]byte, pLen, dLen uint64)
TEXT ·gcmAesFinish(SB),NOSPLIT,$0
#define pTbl R0
#define tMsk R1
#define tPtr R2
#define plen R3
#define dlen R4

	MOVD	$0xC2, R1
	LSL	$56, R1
	MOVD	$1, R0
	VMOV	R1, POLY.D[0]
	VMOV	R0, POLY.D[1]
	VEOR	ZERO.B16, ZERO.B16, ZERO.B16

	MOVD	productTable+0(FP), pTbl
	MOVD	tagMask+8(FP), tMsk
	MOVD	T+16(FP), tPtr
	MOVD	pLen+24(FP), plen
	MOVD	dLen+32(FP), dlen

	VLD1	(tPtr), [ACC0.B16]
	VLD1	(tMsk), [B1.B16]

	LSL	$3, plen
	LSL	$3, dlen

	VMOV	dlen, B0.D[0]
	VMOV	plen, B0.D[1]

	ADD	$14*16, pTbl
	VLD1.P	(pTbl), [T1.B16, T2.B16]

	VEOR	ACC0.B16, B0.B16, B0.B16

	VEXT	$8, B0.B16, B0.B16, T0.B16
	VEOR	B0.B16, T0.B16, T0.B16
	VPMULL	B0.D1, T1.D1, ACC1.Q1
	VPMULL2	B0.D2, T1.D2, ACC0.Q1
	VPMULL	T0.D1, T2.D1, ACCM.Q1

	reduce()

	VREV64	ACC0.B16, ACC0.B16
	VEOR	B1.B16, ACC0.B16, ACC0.B16

	VST1	[ACC0.B16], (tPtr)
	RET
#undef pTbl
#undef tMsk
#undef tPtr
#undef plen
#undef dlen

// func gcmAesInit(productTable *[256]byte, ks []uint32)
TEXT ·gcmAesInit(SB),NOSPLIT,$0
#define pTbl R0
#define KS R1
#define NR R2
#define I R3
	MOVD	productTable+0(FP), pTbl
	MOVD	ks_base+8(FP), KS
	MOVD	ks_len+16(FP), NR

	MOVD	$0xC2, I
	LSL	$56, I
	VMOV	I, POLY.D[0]
	MOVD	$1, I
	VMOV	I, POLY.D[1]
	VEOR	ZERO.B16, ZERO.B16, ZERO.B16

	// Encrypt block 0 with the AES key to generate the hash key H
	VLD1.P	64(KS), [T0.B16, T1.B16, T2.B16, T3.B16]
	VEOR	B0.B16, B0.B16, B0.B16
	AESE	T0.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	T1.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	T2.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	T3.B16, B0.B16
	AESMC	B0.B16, B0.B16
	VLD1.P	64(KS), [T0.B16, T1.B16, T2.B16, T3.B16]
	AESE	T0.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	T1.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	T2.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	T3.B16, B0.B16
	AESMC	B0.B16, B0.B16
	TBZ	$4, NR, initEncFinish
	VLD1.P	32(KS), [T0.B16, T1.B16]
	AESE	T0.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	T1.B16, B0.B16
	AESMC	B0.B16, B0.B16
	TBZ	$3, NR, initEncFinish
	VLD1.P	32(KS), [T0.B16, T1.B16]
	AESE	T0.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	T1.B16, B0.B16
	AESMC	B0.B16, B0.B16
initEncFinish:
	VLD1	(KS), [T0.B16, T1.B16, T2.B16]
	AESE	T0.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	T1.B16, B0.B16
	VEOR	T2.B16, B0.B16, B0.B16

	VREV64	B0.B16, B0.B16

	// Multiply by 2 modulo P
	VMOV	B0.D[0], I
	ASR	$63, I
	VMOV	I, T1.D[0]
	VMOV	I, T1.D[1]
	VAND	POLY.B16, T1.B16, T1.B16
	VUSHR	$63, B0.D2, T2.D2
	VEXT	$8, ZERO.B16, T2.B16, T2.B16
	VSHL	$1, B0.D2, B0.D2
	VEOR	T1.B16, B0.B16, B0.B16
	VEOR	T2.B16, B0.B16, B0.B16 // Can avoid this when VSLI is available

	// Karatsuba pre-computation
	VEXT	$8, B0.B16, B0.B16, B1.B16
	VEOR	B0.B16, B1.B16, B1.B16

	ADD	$14*16, pTbl
	VST1	[B0.B16, B1.B16], (pTbl)
	SUB	$2*16, pTbl

	VMOV	B0.B16, B2.B16
	VMOV	B1.B16, B3.B16

	MOVD	$7, I

initLoop:
	// Compute powers of H
	SUBS	$1, I

	VPMULL	B0.D1, B2.D1, T1.Q1
	VPMULL2	B0.D2, B2.D2, T0.Q1
	VPMULL	B1.D1, B3.D1, T2.Q1
	VEOR	T0.B16, T2.B16, T2.B16
	VEOR	T1.B16, T2.B16, T2.B16
	VEXT	$8, ZERO.B16, T2.B16, T3.B16
	VEXT	$8, T2.B16, ZERO.B16, T2.B16
	VEOR	T2.B16, T0.B16, T0.B16
	VEOR	T3.B16, T1.B16, T1.B16
	VPMULL	POLY.D1, T0.D1, T2.Q1
	VEXT	$8, T0.B16, T0.B16, T0.B16
	VEOR	T2.B16, T0.B16, T0.B16
	VPMULL	POLY.D1, T0.D1, T2.Q1
	VEXT	$8, T0.B16, T0.B16, T0.B16
	VEOR	T2.B16, T0.B16, T0.B16
	VEOR	T1.B16, T0.B16, B2.B16
	VMOV	B2.B16, B3.B16
	VEXT	$8, B2.B16, B2.B16, B2.B16
	VEOR	B2.B16, B3.B16, B3.B16

	VST1	[B2.B16, B3.B16], (pTbl)
	SUB	$2*16, pTbl

	BNE	initLoop
	RET
#undef I
#undef NR
#undef KS
#undef pTbl

// func gcmAesData(productTable *[256]byte, data []byte, T *[16]byte)
TEXT ·gcmAesData(SB),NOSPLIT,$0
#define pTbl R0
#define aut R1
#define tPtr R2
#define autLen R3
#define H0 R4
#define pTblSave R5

#define mulRound(X) \
	VLD1.P	32(pTbl), [T1.B16, T2.B16] \
	VREV64	X.B16, X.B16               \
	VEXT	$8, X.B16, X.B16, T0.B16   \
	VEOR	X.B16, T0.B16, T0.B16      \
	VPMULL	X.D1, T1.D1, T3.Q1         \
	VEOR	T3.B16, ACC1.B16, ACC1.B16 \
	VPMULL2	X.D2, T1.D2, T3.Q1         \
	VEOR	T3.B16, ACC0.B16, ACC0.B16 \
	VPMULL	T0.D1, T2.D1, T3.Q1        \
	VEOR	T3.B16, ACCM.B16, ACCM.B16

	MOVD	productTable+0(FP), pTbl
	MOVD	data_base+8(FP), aut
	MOVD	data_len+16(FP), autLen
	MOVD	T+32(FP), tPtr

	VEOR	ACC0.B16, ACC0.B16, ACC0.B16
	CBZ	autLen, dataBail

	MOVD	$0xC2, H0
	LSL	$56, H0
	VMOV	H0, POLY.D[0]
	MOVD	$1, H0
	VMOV	H0, POLY.D[1]
	VEOR	ZERO.B16, ZERO.B16, ZERO.B16
	MOVD	pTbl, pTblSave

	CMP	$13, autLen
	BEQ	dataTLS
	CMP	$128, autLen
	BLT	startSinglesLoop
	B	octetsLoop

dataTLS:
	ADD	$14*16, pTbl
	VLD1.P	(pTbl), [T1.B16, T2.B16]
	VEOR	B0.B16, B0.B16, B0.B16

	MOVD	(aut), H0
	VMOV	H0, B0.D[0]
	MOVW	8(aut), H0
	VMOV	H0, B0.S[2]
	MOVB	12(aut), H0
	VMOV	H0, B0.B[12]

	MOVD	$0, autLen
	B	dataMul

octetsLoop:
		CMP	$128, autLen
		BLT	startSinglesLoop
		SUB	$128, autLen

		VLD1.P	32(aut), [B0.B16, B1.B16]

		VLD1.P	32(pTbl), [T1.B16, T2.B16]
		VREV64	B0.B16, B0.B16
		VEOR	ACC0.B16, B0.B16, B0.B16
		VEXT	$8, B0.B16, B0.B16, T0.B16
		VEOR	B0.B16, T0.B16, T0.B16
		VPMULL	B0.D1, T1.D1, ACC1.Q1
		VPMULL2	B0.D2, T1.D2, ACC0.Q1
		VPMULL	T0.D1, T2.D1, ACCM.Q1

		mulRound(B1)
		VLD1.P  32(aut), [B2.B16, B3.B16]
		mulRound(B2)
		mulRound(B3)
		VLD1.P  32(aut), [B4.B16, B5.B16]
		mulRound(B4)
		mulRound(B5)
		VLD1.P  32(aut), [B6.B16, B7.B16]
		mulRound(B6)
		mulRound(B7)

		MOVD	pTblSave, pTbl
		reduce()
	B	octetsLoop

startSinglesLoop:

	ADD	$14*16, pTbl
	VLD1.P	(pTbl), [T1.B16, T2.B16]

singlesLoop:

		CMP	$16, autLen
		BLT	dataEnd
		SUB	$16, autLen

		VLD1.P	16(aut), [B0.B16]
dataMul:
		VREV64	B0.B16, B0.B16
		VEOR	ACC0.B16, B0.B16, B0.B16

		VEXT	$8, B0.B16, B0.B16, T0.B16
		VEOR	B0.B16, T0.B16, T0.B16
		VPMULL	B0.D1, T1.D1, ACC1.Q1
		VPMULL2	B0.D2, T1.D2, ACC0.Q1
		VPMULL	T0.D1, T2.D1, ACCM.Q1

		reduce()

	B	singlesLoop

dataEnd:

	CBZ	autLen, dataBail
	VEOR	B0.B16, B0.B16, B0.B16
	ADD	autLen, aut

dataLoadLoop:
		MOVB.W	-1(aut), H0
		VEXT	$15, B0.B16, ZERO.B16, B0.B16
		VMOV	H0, B0.B[0]
		SUBS	$1, autLen
		BNE	dataLoadLoop
	B	dataMul

dataBail:
	VST1	[ACC0.B16], (tPtr)
	RET

#undef pTbl
#undef aut
#undef tPtr
#undef autLen
#undef H0
#undef pTblSave

// func gcmAesEnc(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32)
TEXT ·gcmAesEnc(SB),NOSPLIT,$0
#define pTbl R0
#define dstPtr R1
#define ctrPtr R2
#define srcPtr R3
#define ks R4
#define tPtr R5
#define srcPtrLen R6
#define aluCTR R7
#define aluTMP R8
#define aluK R9
#define NR R10
#define H0 R11
#define H1 R12
#define curK R13
#define pTblSave R14

#define aesrndx8(K) \
	AESE	K.B16, B0.B16    \
	AESMC	B0.B16, B0.B16   \
	AESE	K.B16, B1.B16    \
	AESMC	B1.B16, B1.B16   \
	AESE	K.B16, B2.B16    \
	AESMC	B2.B16, B2.B16   \
	AESE	K.B16, B3.B16    \
	AESMC	B3.B16, B3.B16   \
	AESE	K.B16, B4.B16    \
	AESMC	B4.B16, B4.B16   \
	AESE	K.B16, B5.B16    \
	AESMC	B5.B16, B5.B16   \
	AESE	K.B16, B6.B16    \
	AESMC	B6.B16, B6.B16   \
	AESE	K.B16, B7.B16    \
	AESMC	B7.B16, B7.B16

#define aesrndlastx8(K) \
	AESE	K.B16, B0.B16    \
	AESE	K.B16, B1.B16    \
	AESE	K.B16, B2.B16    \
	AESE	K.B16, B3.B16    \
	AESE	K.B16, B4.B16    \
	AESE	K.B16, B5.B16    \
	AESE	K.B16, B6.B16    \
	AESE	K.B16, B7.B16

	MOVD	productTable+0(FP), pTbl
	MOVD	dst+8(FP), dstPtr
	MOVD	src_base+32(FP), srcPtr
	MOVD	src_len+40(FP), srcPtrLen
	MOVD	ctr+56(FP), ctrPtr
	MOVD	T+64(FP), tPtr
	MOVD	ks_base+72(FP), ks
	MOVD	ks_len+80(FP), NR

	MOVD	$0xC2, H1
	LSL	$56, H1
	MOVD	$1, H0
	VMOV	H1, POLY.D[0]
	VMOV	H0, POLY.D[1]
	VEOR	ZERO.B16, ZERO.B16, ZERO.B16
	// Compute NR from len(ks)
	MOVD	pTbl, pTblSave
	// Current tag, after AAD
	VLD1	(tPtr), [ACC0.B16]
	VEOR	ACC1.B16, ACC1.B16, ACC1.B16
	VEOR	ACCM.B16, ACCM.B16, ACCM.B16
	// Prepare initial counter, and the increment vector
	VLD1	(ctrPtr), [CTR.B16]
	VEOR	INC.B16, INC.B16, INC.B16
	MOVD	$1, H0
	VMOV	H0, INC.S[3]
	VREV32	CTR.B16, CTR.B16
	VADD	CTR.S4, INC.S4, CTR.S4
	// Skip to <8 blocks loop
	CMP	$128, srcPtrLen

	MOVD	ks, H0
	// For AES-128 round keys are stored in: K0 .. K10, KLAST
	VLD1.P	64(H0), [K0.B16, K1.B16, K2.B16, K3.B16]
	VLD1.P	64(H0), [K4.B16, K5.B16, K6.B16, K7.B16]
	VLD1.P	48(H0), [K8.B16, K9.B16, K10.B16]
	VMOV	K10.B16, KLAST.B16

	BLT	startSingles
	// There are at least 8 blocks to encrypt
	TBZ	$4, NR, octetsLoop

	// For AES-192 round keys occupy: K0 .. K7, K10, K11, K8, K9, KLAST
	VMOV	K8.B16, K10.B16
	VMOV	K9.B16, K11.B16
	VMOV	KLAST.B16, K8.B16
	VLD1.P	16(H0), [K9.B16]
	VLD1.P  16(H0), [KLAST.B16]
	TBZ	$3, NR, octetsLoop
	// For AES-256 round keys occupy: K0 .. K7, K10, K11, mem, mem, K8, K9, KLAST
	VMOV	KLAST.B16, K8.B16
	VLD1.P	16(H0), [K9.B16]
	VLD1.P  16(H0), [KLAST.B16]
	ADD	$10*16, ks, H0
	MOVD	H0, curK

octetsLoop:
		SUB	$128, srcPtrLen

		VMOV	CTR.B16, B0.B16
		VADD	B0.S4, INC.S4, B1.S4
		VREV32	B0.B16, B0.B16
		VADD	B1.S4, INC.S4, B2.S4
		VREV32	B1.B16, B1.B16
		VADD	B2.S4, INC.S4, B3.S4
		VREV32	B2.B16, B2.B16
		VADD	B3.S4, INC.S4, B4.S4
		VREV32	B3.B16, B3.B16
		VADD	B4.S4, INC.S4, B5.S4
		VREV32	B4.B16, B4.B16
		VADD	B5.S4, INC.S4, B6.S4
		VREV32	B5.B16, B5.B16
		VADD	B6.S4, INC.S4, B7.S4
		VREV32	B6.B16, B6.B16
		VADD	B7.S4, INC.S4, CTR.S4
		VREV32	B7.B16, B7.B16

		aesrndx8(K0)
		aesrndx8(K1)
		aesrndx8(K2)
		aesrndx8(K3)
		aesrndx8(K4)
		aesrndx8(K5)
		aesrndx8(K6)
		aesrndx8(K7)
		TBZ	$4, NR, octetsFinish
		aesrndx8(K10)
		aesrndx8(K11)
		TBZ	$3, NR, octetsFinish
		VLD1.P	32(curK), [T1.B16, T2.B16]
		aesrndx8(T1)
		aesrndx8(T2)
		MOVD	H0, curK
octetsFinish:
		aesrndx8(K8)
		aesrndlastx8(K9)

		VEOR	KLAST.B16, B0.B16, B0.B16
		VEOR	KLAST.B16, B1.B16, B1.B16
		VEOR	KLAST.B16, B2.B16, B2.B16
		VEOR	KLAST.B16, B3.B16, B3.B16
		VEOR	KLAST.B16, B4.B16, B4.B16
		VEOR	KLAST.B16, B5.B16, B5.B16
		VEOR	KLAST.B16, B6.B16, B6.B16
		VEOR	KLAST.B16, B7.B16, B7.B16

		VLD1.P	32(srcPtr), [T1.B16, T2.B16]
		VEOR	B0.B16, T1.B16, B0.B16
		VEOR	B1.B16, T2.B16, B1.B16
		VST1.P  [B0.B16, B1.B16], 32(dstPtr)
		VLD1.P	32(srcPtr), [T1.B16, T2.B16]
		VEOR	B2.B16, T1.B16, B2.B16
		VEOR	B3.B16, T2.B16, B3.B16
		VST1.P  [B2.B16, B3.B16], 32(dstPtr)
		VLD1.P	32(srcPtr), [T1.B16, T2.B16]
		VEOR	B4.B16, T1.B16, B4.B16
		VEOR	B5.B16, T2.B16, B5.B16
		VST1.P  [B4.B16, B5.B16], 32(dstPtr)
		VLD1.P	32(srcPtr), [T1.B16, T2.B16]
		VEOR	B6.B16, T1.B16, B6.B16
		VEOR	B7.B16, T2.B16, B7.B16
		VST1.P  [B6.B16, B7.B16], 32(dstPtr)

		VLD1.P	32(pTbl), [T1.B16, T2.B16]
		VREV64	B0.B16, B0.B16
		VEOR	ACC0.B16, B0.B16, B0.B16
		VEXT	$8, B0.B16, B0.B16, T0.B16
		VEOR	B0.B16, T0.B16, T0.B16
		VPMULL	B0.D1, T1.D1, ACC1.Q1
		VPMULL2	B0.D2, T1.D2, ACC0.Q1
		VPMULL	T0.D1, T2.D1, ACCM.Q1

		mulRound(B1)
		mulRound(B2)
		mulRound(B3)
		mulRound(B4)
		mulRound(B5)
		mulRound(B6)
		mulRound(B7)
		MOVD	pTblSave, pTbl
		reduce()

		CMP	$128, srcPtrLen
		BGE	octetsLoop

startSingles:
	CBZ	srcPtrLen, done
	ADD	$14*16, pTbl
	// Preload H and its Karatsuba precomp
	VLD1.P	(pTbl), [T1.B16, T2.B16]
	// Preload AES round keys
	ADD	$128, ks
	VLD1.P	48(ks), [K8.B16, K9.B16, K10.B16]
	VMOV	K10.B16, KLAST.B16
	TBZ	$4, NR, singlesLoop
	VLD1.P	32(ks), [B1.B16, B2.B16]
	VMOV	B2.B16, KLAST.B16
	TBZ	$3, NR, singlesLoop
	VLD1.P	32(ks), [B3.B16, B4.B16]
	VMOV	B4.B16, KLAST.B16

singlesLoop:
		CMP	$16, srcPtrLen
		BLT	tail
		SUB	$16, srcPtrLen

		VLD1.P	16(srcPtr), [T0.B16]
		VEOR	KLAST.B16, T0.B16, T0.B16

		VREV32	CTR.B16, B0.B16
		VADD	CTR.S4, INC.S4, CTR.S4

		AESE	K0.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K1.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K2.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K3.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K4.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K5.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K6.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K7.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K8.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K9.B16, B0.B16
		TBZ	$4, NR, singlesLast
		AESMC	B0.B16, B0.B16
		AESE	K10.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	B1.B16, B0.B16
		TBZ	$3, NR, singlesLast
		AESMC	B0.B16, B0.B16
		AESE	B2.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	B3.B16, B0.B16
singlesLast:
		VEOR	T0.B16, B0.B16, B0.B16
encReduce:
		VST1.P	[B0.B16], 16(dstPtr)

		VREV64	B0.B16, B0.B16
		VEOR	ACC0.B16, B0.B16, B0.B16

		VEXT	$8, B0.B16, B0.B16, T0.B16
		VEOR	B0.B16, T0.B16, T0.B16
		VPMULL	B0.D1, T1.D1, ACC1.Q1
		VPMULL2	B0.D2, T1.D2, ACC0.Q1
		VPMULL	T0.D1, T2.D1, ACCM.Q1

		reduce()

	B	singlesLoop
tail:
	CBZ	srcPtrLen, done

	VEOR	T0.B16, T0.B16, T0.B16
	VEOR	T3.B16, T3.B16, T3.B16
	MOVD	$0, H1
	SUB	$1, H1
	ADD	srcPtrLen, srcPtr

	TBZ	$3, srcPtrLen, ld4
	MOVD.W	-8(srcPtr), H0
	VMOV	H0, T0.D[0]
	VMOV	H1, T3.D[0]
ld4:
	TBZ	$2, srcPtrLen, ld2
	MOVW.W	-4(srcPtr), H0
	VEXT	$12, T0.B16, ZERO.B16, T0.B16
	VEXT	$12, T3.B16, ZERO.B16, T3.B16
	VMOV	H0, T0.S[0]
	VMOV	H1, T3.S[0]
ld2:
	TBZ	$1, srcPtrLen, ld1
	MOVH.W	-2(srcPtr), H0
	VEXT	$14, T0.B16, ZERO.B16, T0.B16
	VEXT	$14, T3.B16, ZERO.B16, T3.B16
	VMOV	H0, T0.H[0]
	VMOV	H1, T3.H[0]
ld1:
	TBZ	$0, srcPtrLen, ld0
	MOVB.W	-1(srcPtr), H0
	VEXT	$15, T0.B16, ZERO.B16, T0.B16
	VEXT	$15, T3.B16, ZERO.B16, T3.B16
	VMOV	H0, T0.B[0]
	VMOV	H1, T3.B[0]
ld0:

	MOVD	ZR, srcPtrLen
	VEOR	KLAST.B16, T0.B16, T0.B16
	VREV32	CTR.B16, B0.B16

	AESE	K0.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K1.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K2.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K3.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K4.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K5.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K6.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K7.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K8.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K9.B16, B0.B16
	TBZ	$4, NR, tailLast
	AESMC	B0.B16, B0.B16
	AESE	K10.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	B1.B16, B0.B16
	TBZ	$3, NR, tailLast
	AESMC	B0.B16, B0.B16
	AESE	B2.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	B3.B16, B0.B16

tailLast:
	VEOR	T0.B16, B0.B16, B0.B16
	VAND	T3.B16, B0.B16, B0.B16
	B	encReduce

done:
	VST1	[ACC0.B16], (tPtr)
	RET

// func gcmAesDec(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32)
TEXT ·gcmAesDec(SB),NOSPLIT,$0
	MOVD	productTable+0(FP), pTbl
	MOVD	dst+8(FP), dstPtr
	MOVD	src_base+32(FP), srcPtr
	MOVD	src_len+40(FP), srcPtrLen
	MOVD	ctr+56(FP), ctrPtr
	MOVD	T+64(FP), tPtr
	MOVD	ks_base+72(FP), ks
	MOVD	ks_len+80(FP), NR

	MOVD	$0xC2, H1
	LSL	$56, H1
	MOVD	$1, H0
	VMOV	H1, POLY.D[0]
	VMOV	H0, POLY.D[1]
	VEOR	ZERO.B16, ZERO.B16, ZERO.B16
	// Compute NR from len(ks)
	MOVD	pTbl, pTblSave
	// Current tag, after AAD
	VLD1	(tPtr), [ACC0.B16]
	VEOR	ACC1.B16, ACC1.B16, ACC1.B16
	VEOR	ACCM.B16, ACCM.B16, ACCM.B16
	// Prepare initial counter, and the increment vector
	VLD1	(ctrPtr), [CTR.B16]
	VEOR	INC.B16, INC.B16, INC.B16
	MOVD	$1, H0
	VMOV	H0, INC.S[3]
	VREV32	CTR.B16, CTR.B16
	VADD	CTR.S4, INC.S4, CTR.S4

	MOVD	ks, H0
	// For AES-128 round keys are stored in: K0 .. K10, KLAST
	VLD1.P	64(H0), [K0.B16, K1.B16, K2.B16, K3.B16]
	VLD1.P	64(H0), [K4.B16, K5.B16, K6.B16, K7.B16]
	VLD1.P	48(H0), [K8.B16, K9.B16, K10.B16]
	VMOV	K10.B16, KLAST.B16

	// Skip to <8 blocks loop
	CMP	$128, srcPtrLen
	BLT	startSingles
	// There are at least 8 blocks to encrypt
	TBZ	$4, NR, octetsLoop

	// For AES-192 round keys occupy: K0 .. K7, K10, K11, K8, K9, KLAST
	VMOV	K8.B16, K10.B16
	VMOV	K9.B16, K11.B16
	VMOV	KLAST.B16, K8.B16
	VLD1.P	16(H0), [K9.B16]
	VLD1.P  16(H0), [KLAST.B16]
	TBZ	$3, NR, octetsLoop
	// For AES-256 round keys occupy: K0 .. K7, K10, K11, mem, mem, K8, K9, KLAST
	VMOV	KLAST.B16, K8.B16
	VLD1.P	16(H0), [K9.B16]
	VLD1.P  16(H0), [KLAST.B16]
	ADD	$10*16, ks, H0
	MOVD	H0, curK

octetsLoop:
		SUB	$128, srcPtrLen

		VMOV	CTR.B16, B0.B16
		VADD	B0.S4, INC.S4, B1.S4
		VREV32	B0.B16, B0.B16
		VADD	B1.S4, INC.S4, B2.S4
		VREV32	B1.B16, B1.B16
		VADD	B2.S4, INC.S4, B3.S4
		VREV32	B2.B16, B2.B16
		VADD	B3.S4, INC.S4, B4.S4
		VREV32	B3.B16, B3.B16
		VADD	B4.S4, INC.S4, B5.S4
		VREV32	B4.B16, B4.B16
		VADD	B5.S4, INC.S4, B6.S4
		VREV32	B5.B16, B5.B16
		VADD	B6.S4, INC.S4, B7.S4
		VREV32	B6.B16, B6.B16
		VADD	B7.S4, INC.S4, CTR.S4
		VREV32	B7.B16, B7.B16

		aesrndx8(K0)
		aesrndx8(K1)
		aesrndx8(K2)
		aesrndx8(K3)
		aesrndx8(K4)
		aesrndx8(K5)
		aesrndx8(K6)
		aesrndx8(K7)
		TBZ	$4, NR, octetsFinish
		aesrndx8(K10)
		aesrndx8(K11)
		TBZ	$3, NR, octetsFinish
		VLD1.P	32(curK), [T1.B16, T2.B16]
		aesrndx8(T1)
		aesrndx8(T2)
		MOVD	H0, curK
octetsFinish:
		aesrndx8(K8)
		aesrndlastx8(K9)

		VEOR	KLAST.B16, B0.B16, T1.B16
		VEOR	KLAST.B16, B1.B16, T2.B16
		VEOR	KLAST.B16, B2.B16, B2.B16
		VEOR	KLAST.B16, B3.B16, B3.B16
		VEOR	KLAST.B16, B4.B16, B4.B16
		VEOR	KLAST.B16, B5.B16, B5.B16
		VEOR	KLAST.B16, B6.B16, B6.B16
		VEOR	KLAST.B16, B7.B16, B7.B16

		VLD1.P	32(srcPtr), [B0.B16, B1.B16]
		VEOR	B0.B16, T1.B16, T1.B16
		VEOR	B1.B16, T2.B16, T2.B16
		VST1.P  [T1.B16, T2.B16], 32(dstPtr)

		VLD1.P	32(pTbl), [T1.B16, T2.B16]
		VREV64	B0.B16, B0.B16
		VEOR	ACC0.B16, B0.B16, B0.B16
		VEXT	$8, B0.B16, B0.B16, T0.B16
		VEOR	B0.B16, T0.B16, T0.B16
		VPMULL	B0.D1, T1.D1, ACC1.Q1
		VPMULL2	B0.D2, T1.D2, ACC0.Q1
		VPMULL	T0.D1, T2.D1, ACCM.Q1
		mulRound(B1)

		VLD1.P	32(srcPtr), [B0.B16, B1.B16]
		VEOR	B2.B16, B0.B16, T1.B16
		VEOR	B3.B16, B1.B16, T2.B16
		VST1.P  [T1.B16, T2.B16], 32(dstPtr)
		mulRound(B0)
		mulRound(B1)

		VLD1.P	32(srcPtr), [B0.B16, B1.B16]
		VEOR	B4.B16, B0.B16, T1.B16
		VEOR	B5.B16, B1.B16, T2.B16
		VST1.P  [T1.B16, T2.B16], 32(dstPtr)
		mulRound(B0)
		mulRound(B1)

		VLD1.P	32(srcPtr), [B0.B16, B1.B16]
		VEOR	B6.B16, B0.B16, T1.B16
		VEOR	B7.B16, B1.B16, T2.B16
		VST1.P  [T1.B16, T2.B16], 32(dstPtr)
		mulRound(B0)
		mulRound(B1)

		MOVD	pTblSave, pTbl
		reduce()

		CMP	$128, srcPtrLen
		BGE	octetsLoop

startSingles:
	CBZ	srcPtrLen, done
	ADD	$14*16, pTbl
	// Preload H and its Karatsuba precomp
	VLD1.P	(pTbl), [T1.B16, T2.B16]
	// Preload AES round keys
	ADD	$128, ks
	VLD1.P	48(ks), [K8.B16, K9.B16, K10.B16]
	VMOV	K10.B16, KLAST.B16
	TBZ	$4, NR, singlesLoop
	VLD1.P	32(ks), [B1.B16, B2.B16]
	VMOV	B2.B16, KLAST.B16
	TBZ	$3, NR, singlesLoop
	VLD1.P	32(ks), [B3.B16, B4.B16]
	VMOV	B4.B16, KLAST.B16

singlesLoop:
		CMP	$16, srcPtrLen
		BLT	tail
		SUB	$16, srcPtrLen

		VLD1.P	16(srcPtr), [T0.B16]
		VREV64	T0.B16, B5.B16
		VEOR	KLAST.B16, T0.B16, T0.B16

		VREV32	CTR.B16, B0.B16
		VADD	CTR.S4, INC.S4, CTR.S4

		AESE	K0.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K1.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K2.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K3.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K4.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K5.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K6.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K7.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K8.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	K9.B16, B0.B16
		TBZ	$4, NR, singlesLast
		AESMC	B0.B16, B0.B16
		AESE	K10.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	B1.B16, B0.B16
		TBZ	$3, NR, singlesLast
		AESMC	B0.B16, B0.B16
		AESE	B2.B16, B0.B16
		AESMC	B0.B16, B0.B16
		AESE	B3.B16, B0.B16
singlesLast:
		VEOR	T0.B16, B0.B16, B0.B16

		VST1.P	[B0.B16], 16(dstPtr)

		VEOR	ACC0.B16, B5.B16, B5.B16
		VEXT	$8, B5.B16, B5.B16, T0.B16
		VEOR	B5.B16, T0.B16, T0.B16
		VPMULL	B5.D1, T1.D1, ACC1.Q1
		VPMULL2	B5.D2, T1.D2, ACC0.Q1
		VPMULL	T0.D1, T2.D1, ACCM.Q1
		reduce()

	B	singlesLoop
tail:
	CBZ	srcPtrLen, done

	VREV32	CTR.B16, B0.B16
	VADD	CTR.S4, INC.S4, CTR.S4

	AESE	K0.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K1.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K2.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K3.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K4.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K5.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K6.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K7.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K8.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	K9.B16, B0.B16
	TBZ	$4, NR, tailLast
	AESMC	B0.B16, B0.B16
	AESE	K10.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	B1.B16, B0.B16
	TBZ	$3, NR, tailLast
	AESMC	B0.B16, B0.B16
	AESE	B2.B16, B0.B16
	AESMC	B0.B16, B0.B16
	AESE	B3.B16, B0.B16
tailLast:
	VEOR	KLAST.B16, B0.B16, B0.B16

	// Assuming it is safe to load past dstPtr due to the presence of the tag
	VLD1	(srcPtr), [B5.B16]

	VEOR	B5.B16, B0.B16, B0.B16

	VEOR	T3.B16, T3.B16, T3.B16
	MOVD	$0, H1
	SUB	$1, H1

	TBZ	$3, srcPtrLen, ld4
	VMOV	B0.D[0], H0
	MOVD.P	H0, 8(dstPtr)
	VMOV	H1, T3.D[0]
	VEXT	$8, ZERO.B16, B0.B16, B0.B16
ld4:
	TBZ	$2, srcPtrLen, ld2
	VMOV	B0.S[0], H0
	MOVW.P	H0, 4(dstPtr)
	VEXT	$12, T3.B16, ZERO.B16, T3.B16
	VMOV	H1, T3.S[0]
	VEXT	$4, ZERO.B16, B0.B16, B0.B16
ld2:
	TBZ	$1, srcPtrLen, ld1
	VMOV	B0.H[0], H0
	MOVH.P	H0, 2(dstPtr)
	VEXT	$14, T3.B16, ZERO.B16, T3.B16
	VMOV	H1, T3.H[0]
	VEXT	$2, ZERO.B16, B0.B16, B0.B16
ld1:
	TBZ	$0, srcPtrLen, ld0
	VMOV	B0.B[0], H0
	MOVB.P	H0, 1(dstPtr)
	VEXT	$15, T3.B16, ZERO.B16, T3.B16
	VMOV	H1, T3.B[0]
ld0:

	VAND	T3.B16, B5.B16, B5.B16
	VREV64	B5.B16, B5.B16

	VEOR	ACC0.B16, B5.B16, B5.B16
	VEXT	$8, B5.B16, B5.B16, T0.B16
	VEOR	B5.B16, T0.B16, T0.B16
	VPMULL	B5.D1, T1.D1, ACC1.Q1
	VPMULL2	B5.D2, T1.D2, ACC0.Q1
	VPMULL	T0.D1, T2.D1, ACCM.Q1
	reduce()
done:
	VST1	[ACC0.B16], (tPtr)

	RET