crypto/internal/fips140/aes: avoid stack round-trip when building counter blocks on loong64

ctrBlocks1/2/4/8Asm previously built each 128-bit counter block by
storing the two big-endian 64-bit halves to a stack scratch buffer
and then reloading them with VMOVQ. Replace this with VMOVQ Rj,
Vd.V[index] (vinsgr2vr.d), which writes each half directly into the
target vector register lane, eliminating the store-to-load round
trip. This also lets the stack frame for these four functions shrink
from $16/$32 to $0.

goos: linux
goarch: loong64
pkg: crypto/cipher
cpu: Loongson-3C6000/S @ 2200.00MHz
              | old.bench.aesctr |          new.bench.aesctr           |
              |      sec/op      |   sec/op     vs base                |
AESCTR/128/50        363.9n ± 0%   327.1n ± 0%  -10.11% (p=0.000 n=10)
AESCTR/128/1K        4.310µ ± 0%   3.963µ ± 0%   -8.05% (p=0.000 n=10)
AESCTR/128/8K        33.25µ ± 0%   30.57µ ± 0%   -8.06% (p=0.000 n=10)
AESCTR/192/50        424.8n ± 0%   388.7n ± 0%   -8.50% (p=0.000 n=10)
AESCTR/192/1K        5.090µ ± 0%   4.742µ ± 0%   -6.84% (p=0.000 n=10)
AESCTR/192/8K        39.30µ ± 0%   36.60µ ± 0%   -6.87% (p=0.000 n=10)
AESCTR/256/50        485.4n ± 0%   450.0n ± 0%   -7.29% (p=0.000 n=10)
AESCTR/256/1K        5.861µ ± 0%   5.516µ ± 0%   -5.89% (p=0.000 n=10)
AESCTR/256/8K        45.28µ ± 0%   42.60µ ± 0%   -5.92% (p=0.000 n=10)
geomean              4.361µ        4.034µ        -7.51%

goos: linux
goarch: loong64
pkg: crypto/cipher
cpu: Loongson-3A5000 @ 2500.00MHz
              | old.bench.aesctr |          new.bench.aesctr          |
              |      sec/op      |   sec/op     vs base               |
AESCTR/128/50        407.7n ± 0%   387.1n ± 0%  -5.05% (p=0.000 n=10)
AESCTR/128/1K        5.232µ ± 0%   4.925µ ± 0%  -5.87% (p=0.000 n=10)
AESCTR/128/8K        40.58µ ± 0%   38.14µ ± 0%  -6.03% (p=0.000 n=10)
AESCTR/192/50        477.8n ± 0%   453.7n ± 0%  -5.04% (p=0.000 n=10)
AESCTR/192/1K        6.201µ ± 0%   5.890µ ± 0%  -5.02% (p=0.000 n=10)
AESCTR/192/8K        48.15µ ± 0%   45.67µ ± 0%  -5.14% (p=0.000 n=10)
AESCTR/256/50        543.5n ± 0%   522.5n ± 0%  -3.86% (p=0.000 n=10)
AESCTR/256/1K        7.161µ ± 0%   6.852µ ± 0%  -4.32% (p=0.000 n=10)
AESCTR/256/8K        55.62µ ± 1%   53.19µ ± 0%  -4.38% (p=0.000 n=10)
geomean              5.177µ        4.920µ       -4.97%

Change-Id: I6fc973db36454b2e94272b09a0ee93e3842fe950
Reviewed-on: https://go-review.googlesource.com/c/go/+/835565
Reviewed-by: abner chenc <chenguoqi@loongson.cn>
LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Michael Pratt <mpratt@google.com>
Reviewed-by: Mark Freeman <mark@golang.org>

src/crypto/internal/fips140/aes/ctr_loong64.s

diff --git a/src/crypto/internal/fips140/aes/ctr_loong64.s b/src/crypto/internal/fips140/aes/ctr_loong64.s
index 38a7192..fcee0cc 100644
--- a/src/crypto/internal/fips140/aes/ctr_loong64.s
+++ b/src/crypto/internal/fips140/aes/ctr_loong64.s
@@ -100,7 +100,7 @@
 	VXORV	out, t1, out		/* out ^= xtime(...) */
 
 // func ctrBlocks1Asm(nr int, xk *[60]uint32, dst, src *[BlockSize]byte, ivlo, ivhi uint64)
-TEXT ·ctrBlocks1Asm(SB), NOSPLIT, $16-48
+TEXT ·ctrBlocks1Asm(SB), NOSPLIT, $0-48
 	MOVV	nr+0(FP), R4
 	MOVV	xk+8(FP), R5
 	MOVV	dst+16(FP), R6
@@ -108,12 +108,11 @@
 	MOVV	ivlo+32(FP), R8
 	MOVV	ivhi+40(FP), R9
 
-	// Construct a 128-bit counter block onto the stack, big endian [ivhi][ivlo]
+	// Construct a 128-bit counter block directly in V0, big endian [ivhi][ivlo]
 	REVBV	R9, R10		// ivhi little -> big
 	REVBV	R8, R11		// ivlo little -> big
-	MOVV	R10, 0(R3)
-	MOVV	R11, 8(R3)
-	VMOVQ	(R3), V0	// V0 = counter block
+	VMOVQ	R10, V0.V[0]	// V0 = counter block
+	VMOVQ	R11, V0.V[1]
 
 	MOVV	$shiftRows(SB), R12
 	VMOVQ	(R12), V7
@@ -167,7 +166,7 @@
 	RET
 
 // func ctrBlocks2Asm(nr int, xk *[60]uint32, dst, src *[2*BlockSize]byte, ivlo, ivhi uint64)
-TEXT ·ctrBlocks2Asm(SB), NOSPLIT, $32-48
+TEXT ·ctrBlocks2Asm(SB), NOSPLIT, $0-48
 	MOVV	nr+0(FP), R4
 	MOVV	xk+8(FP), R5
 	MOVV	dst+16(FP), R6
@@ -178,9 +177,8 @@
 	// Build counter 0 = (ivlo, ivhi)
 	REVBV	R9, R10
 	REVBV	R8, R11
-	MOVV	R10, 0(R3)
-	MOVV	R11, 8(R3)
-	VMOVQ	(R3), V0
+	VMOVQ	R10, V0.V[0]
+	VMOVQ	R11, V0.V[1]
 
 	// Build counter 1 = (ivlo+1, ivhi + carry)
 	MOVV	R8, R16
@@ -189,9 +187,8 @@
 	ADDV	R9, R18, R19	// R19 = ivhi + carry
 	REVBV	R19, R10
 	REVBV	R17, R11
-	MOVV	R10, 16(R3)
-	MOVV	R11, 24(R3)
-	VMOVQ	16(R3), V1
+	VMOVQ	R10, V1.V[0]
+	VMOVQ	R11, V1.V[1]
 
 	// Load persistent constant tables (same as ctrBlocks1Asm)
 	MOVV	$shiftRows(SB), R12
@@ -260,7 +257,7 @@
 	RET
 
 // func ctrBlocks4Asm(nr int, xk *[60]uint32, dst, src *[4*BlockSize]byte, ivlo, ivhi uint64)
-TEXT ·ctrBlocks4Asm(SB), NOSPLIT, $16-48
+TEXT ·ctrBlocks4Asm(SB), NOSPLIT, $0-48
 	MOVV	nr+0(FP), R4
 	MOVV	xk+8(FP), R5
 	MOVV	dst+16(FP), R6
@@ -271,9 +268,8 @@
 	// Construct 4 consecutive counter blocks in sequence (big endian), ivlo+1 needs to process carry between blocks
 	REVBV	R9, R10
 	REVBV	R8, R11
-	MOVV	R10, 0(R3)
-	MOVV	R11, 8(R3)
-	VMOVQ	(R3), V0	// block 0
+	VMOVQ	R10, V0.V[0]	// block 0
+	VMOVQ	R11, V0.V[1]
 
 	ADDV	$1, R8, R16	// R16 = ivlo+1
 	SGTU	R16, R8, R17	// R17 = 1 if not overflow (R16 > R8); if overflow R17=0
@@ -281,9 +277,8 @@
 	ADDV	R17, R9, R18	// R18 = ivhi + carry
 	REVBV	R18, R10
 	REVBV	R16, R11
-	MOVV	R10, 0(R3)
-	MOVV	R11, 8(R3)
-	VMOVQ	(R3), V1	// block 1
+	VMOVQ	R10, V1.V[0]	// block 1
+	VMOVQ	R11, V1.V[1]
 
 	ADDV	$1, R16, R23
 	SGTU	R23, R16, R17
@@ -291,9 +286,8 @@
 	ADDV	R17, R18, R24
 	REVBV	R24, R10
 	REVBV	R23, R11
-	MOVV	R10, 0(R3)
-	MOVV	R11, 8(R3)
-	VMOVQ	(R3), V3	// block 2
+	VMOVQ	R10, V3.V[0]	// block 2
+	VMOVQ	R11, V3.V[1]
 
 	ADDV	$1, R23, R25
 	SGTU	R25, R23, R17
@@ -301,9 +295,8 @@
 	ADDV	R17, R24, R26
 	REVBV	R26, R10
 	REVBV	R25, R11
-	MOVV	R10, 0(R3)
-	MOVV	R11, 8(R3)
-	VMOVQ	(R3), V4	// block 3
+	VMOVQ	R10, V4.V[0]	// block 3
+	VMOVQ	R11, V4.V[1]
 
 	MOVV	$shiftRows(SB), R12
 	VMOVQ	(R12), V7
@@ -393,7 +386,7 @@
 	RET
 
 // func ctrBlocks8Asm(nr int, xk *[60]uint32, dst, src *[8*BlockSize]byte, ivlo, ivhi uint64)
-TEXT ·ctrBlocks8Asm(SB), NOSPLIT, $32-48
+TEXT ·ctrBlocks8Asm(SB), NOSPLIT, $0-48
 	MOVV	nr+0(FP), R4
 	MOVV	xk+8(FP), R5
 	MOVV	dst+16(FP), R6
@@ -403,9 +396,8 @@
 
 	REVBV	R9, R10
 	REVBV	R8, R11
-	MOVV	R10, 0(R3)
-	MOVV	R11, 8(R3)
-	VMOVQ	(R3), V0
+	VMOVQ	R10, V0.V[0]
+	VMOVQ	R11, V0.V[1]
 
 	ADDV	$1, R8, R16
 	SGTU	R8, R16, R17
@@ -414,9 +406,8 @@
 	MOVV	R17, R9
 	REVBV	R9, R10
 	REVBV	R8, R11
-	MOVV	R10, 16(R3)
-	MOVV	R11, 24(R3)
-	VMOVQ	16(R3), V1
+	VMOVQ	R10, V1.V[0]
+	VMOVQ	R11, V1.V[1]
 
 	ADDV	$1, R8, R16
 	SGTU	R8, R16, R17
@@ -425,9 +416,8 @@
 	MOVV	R17, R9
 	REVBV	R9, R10
 	REVBV	R8, R11
-	MOVV	R10, 0(R3)
-	MOVV	R11, 8(R3)
-	VMOVQ	(R3), V3
+	VMOVQ	R10, V3.V[0]
+	VMOVQ	R11, V3.V[1]
 
 	ADDV	$1, R8, R16
 	SGTU	R8, R16, R17
@@ -436,9 +426,8 @@
 	MOVV	R17, R9
 	REVBV	R9, R10
 	REVBV	R8, R11
-	MOVV	R10, 16(R3)
-	MOVV	R11, 24(R3)
-	VMOVQ	16(R3), V4
+	VMOVQ	R10, V4.V[0]
+	VMOVQ	R11, V4.V[1]
 
 	ADDV	$1, R8, R16
 	SGTU	R8, R16, R17
@@ -447,9 +436,8 @@
 	MOVV	R17, R9
 	REVBV	R9, R10
 	REVBV	R8, R11
-	MOVV	R10, 0(R3)
-	MOVV	R11, 8(R3)
-	VMOVQ	(R3), V5
+	VMOVQ	R10, V5.V[0]
+	VMOVQ	R11, V5.V[1]
 
 	ADDV	$1, R8, R16
 	SGTU	R8, R16, R17
@@ -458,9 +446,8 @@
 	MOVV	R17, R9
 	REVBV	R9, R10
 	REVBV	R8, R11
-	MOVV	R10, 16(R3)
-	MOVV	R11, 24(R3)
-	VMOVQ	16(R3), V6
+	VMOVQ	R10, V6.V[0]
+	VMOVQ	R11, V6.V[1]
 
 	ADDV	$1, R8, R16
 	SGTU	R8, R16, R17
@@ -469,9 +456,8 @@
 	MOVV	R17, R9
 	REVBV	R9, R10
 	REVBV	R8, R11
-	MOVV	R10, 0(R3)
-	MOVV	R11, 8(R3)
-	VMOVQ	(R3), V8
+	VMOVQ	R10, V8.V[0]
+	VMOVQ	R11, V8.V[1]
 
 	ADDV	$1, R8, R16
 	SGTU	R8, R16, R17
@@ -480,9 +466,8 @@
 	MOVV	R17, R9
 	REVBV	R9, R10
 	REVBV	R8, R11
-	MOVV	R10, 16(R3)
-	MOVV	R11, 24(R3)
-	VMOVQ	16(R3), V9
+	VMOVQ	R10, V9.V[0]
+	VMOVQ	R11, V9.V[1]
 
 	MOVV	$shiftRows(SB), R12
 	VMOVQ	(R12), V7