| // Copyright 2026 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| // derived from chacha_arm64.s |
| |
| //go:build gc && !purego |
| |
| #include "textflag.h" |
| |
| DATA ·constants+0x00(SB)/4, $0x61707865 |
| DATA ·constants+0x04(SB)/4, $0x3320646e |
| DATA ·constants+0x08(SB)/4, $0x79622d32 |
| DATA ·constants+0x0c(SB)/4, $0x6b206574 |
| GLOBL ·constants(SB), NOPTR|RODATA, $32 |
| |
| DATA ·incRotMatrix+0x00(SB)/4, $0x00000000 |
| DATA ·incRotMatrix+0x04(SB)/4, $0x00000001 |
| DATA ·incRotMatrix+0x08(SB)/4, $0x00000002 |
| DATA ·incRotMatrix+0x0c(SB)/4, $0x00000003 |
| GLOBL ·incRotMatrix(SB), NOPTR|RODATA, $32 |
| |
| #define NUM_ROUNDS 10 |
| |
| // func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32) |
| TEXT ·xorKeyStreamVX(SB), NOSPLIT, $0 |
| MOVV dst+0(FP), R4 |
| MOVV src+24(FP), R5 |
| MOVV src_len+32(FP), R6 |
| MOVV key+48(FP), R7 |
| MOVV nonce+56(FP), R8 |
| MOVV counter+64(FP), R9 |
| |
| MOVV $·constants(SB), R10 |
| MOVV $·incRotMatrix(SB), R11 |
| |
| MOVW (R9), R12 |
| |
| loop: |
| MOVV $NUM_ROUNDS, R15 |
| // load 4-32bit data from incRotMatrix added to counter |
| VMOVQ (R11), V30 |
| |
| // load contants |
| VMOVQ (R10), V0.W4 |
| VMOVQ 4(R10), V1.W4 |
| VMOVQ 8(R10), V2.W4 |
| VMOVQ 12(R10), V3.W4 |
| |
| // load keys |
| VMOVQ (R7), V4.W4 |
| VMOVQ 4(R7), V5.W4 |
| VMOVQ 8(R7), V6.W4 |
| VMOVQ 12(R7), V7.W4 |
| VMOVQ 16(R7), V8.W4 |
| VMOVQ 20(R7), V9.W4 |
| VMOVQ 24(R7), V10.W4 |
| VMOVQ 28(R7), V11.W4 |
| |
| // load counter + nonce |
| VMOVQ (R9), V12.W4 |
| |
| VMOVQ (R8), V13.W4 |
| VMOVQ 4(R8), V14.W4 |
| VMOVQ 8(R8), V15.W4 |
| |
| // update counter |
| VADDW V30, V12, V12 |
| |
| chacha: |
| // V0..V3 += V4..V7 |
| // V12..V15 <<<= ((V12..V15 XOR V0..V3), 16) |
| VADDW V0, V4, V0 |
| VADDW V1, V5, V1 |
| VADDW V2, V6, V2 |
| VADDW V3, V7, V3 |
| VXORV V12, V0, V12 |
| VXORV V13, V1, V13 |
| VXORV V14, V2, V14 |
| VXORV V15, V3, V15 |
| VROTRW $16, V12, V12 |
| VROTRW $16, V13, V13 |
| VROTRW $16, V14, V14 |
| VROTRW $16, V15, V15 |
| |
| // V8..V11 += V12..V15 |
| // V4..V7 <<<= ((V4..V7 XOR V8..V11), 12) |
| VADDW V8, V12, V8 |
| VADDW V9, V13, V9 |
| VADDW V10, V14, V10 |
| VADDW V11, V15, V11 |
| VXORV V4, V8, V4 |
| VXORV V5, V9, V5 |
| VXORV V6, V10, V6 |
| VXORV V7, V11, V7 |
| VROTRW $20, V4, V4 |
| VROTRW $20, V5, V5 |
| VROTRW $20, V6, V6 |
| VROTRW $20, V7, V7 |
| |
| // V0..V3 += V4..V7 |
| // V12..V15 <<<= ((V12..V15 XOR V0..V3), 8) |
| VADDW V0, V4, V0 |
| VADDW V1, V5, V1 |
| VADDW V2, V6, V2 |
| VADDW V3, V7, V3 |
| VXORV V12, V0, V12 |
| VXORV V13, V1, V13 |
| VXORV V14, V2, V14 |
| VXORV V15, V3, V15 |
| VROTRW $24, V12, V12 |
| VROTRW $24, V13, V13 |
| VROTRW $24, V14, V14 |
| VROTRW $24, V15, V15 |
| |
| // V8..V11 += V12..V15 |
| // V4..V7 <<<= ((V4..V7 XOR V8..V11), 7) |
| VADDW V12, V8, V8 |
| VADDW V13, V9, V9 |
| VADDW V14, V10, V10 |
| VADDW V15, V11, V11 |
| VXORV V4, V8, V4 |
| VXORV V5, V9, V5 |
| VXORV V6, V10, V6 |
| VXORV V7, V11, V7 |
| VROTRW $25, V4, V4 |
| VROTRW $25, V5, V5 |
| VROTRW $25, V6, V6 |
| VROTRW $25, V7, V7 |
| |
| // V0..V3 += V5..V7, V4 |
| // V15,V12-V14 <<<= ((V15,V12-V14 XOR V0..V3), 16) |
| VADDW V0, V5, V0 |
| VADDW V1, V6, V1 |
| VADDW V2, V7, V2 |
| VADDW V3, V4, V3 |
| VXORV V15, V0, V15 |
| VXORV V12, V1, V12 |
| VXORV V13, V2, V13 |
| VXORV V14, V3, V14 |
| VROTRW $16, V15, V15 |
| VROTRW $16, V12, V12 |
| VROTRW $16, V13, V13 |
| VROTRW $16, V14, V14 |
| |
| // V10,V11,V8,V9 += V15,V12,V13,V14 |
| // V5,V6,V7,V4 <<<= ((V5,V6,V7,V4 XOR V10,V11,V8,V9), 12) |
| VADDW V10, V15, V10 |
| VADDW V11, V12, V11 |
| VADDW V8, V13, V8 |
| VADDW V9, V14, V9 |
| VXORV V5, V10, V5 |
| VXORV V6, V11, V6 |
| VXORV V7, V8, V7 |
| VXORV V4, V9, V4 |
| VROTRW $20, V5, V5 |
| VROTRW $20, V6, V6 |
| VROTRW $20, V7, V7 |
| VROTRW $20, V4, V4 |
| |
| // V0..V3 += V5..V7, V4 |
| // V15,V12-V14 <<<= ((V15,V12-V14 XOR V0..V3), 8) |
| VADDW V5, V0, V0 |
| VADDW V6, V1, V1 |
| VADDW V7, V2, V2 |
| VADDW V4, V3, V3 |
| VXORV V15, V0, V15 |
| VXORV V12, V1, V12 |
| VXORV V13, V2, V13 |
| VXORV V14, V3, V14 |
| VROTRW $24, V15, V15 |
| VROTRW $24, V12, V12 |
| VROTRW $24, V13, V13 |
| VROTRW $24, V14, V14 |
| |
| // V10,V11,V8,V9 += V15,V12,V13,V14 |
| // V5,V6,V7,V4 <<<= ((V5,V6,V7,V4 XOR V10,V11,V8,V9), 7) |
| VADDW V15, V10, V10 |
| VADDW V12, V11, V11 |
| VADDW V13, V8, V8 |
| VADDW V14, V9, V9 |
| VXORV V5, V10, V5 |
| VXORV V6, V11, V6 |
| VXORV V7, V8, V7 |
| VXORV V4, V9, V4 |
| VROTRW $25, V5, V5 |
| VROTRW $25, V6, V6 |
| VROTRW $25, V7, V7 |
| VROTRW $25, V4, V4 |
| |
| SUBV $1, R15 |
| BNE R15, R0, chacha |
| |
| // load origin contants |
| VMOVQ (R10), V16.W4 |
| VMOVQ 4(R10), V17.W4 |
| VMOVQ 8(R10), V18.W4 |
| VMOVQ 12(R10), V19.W4 |
| |
| // load origin keys |
| VMOVQ (R7), V20.W4 |
| VMOVQ 4(R7), V21.W4 |
| VMOVQ 8(R7), V22.W4 |
| VMOVQ 12(R7), V23.W4 |
| VMOVQ 16(R7), V24.W4 |
| VMOVQ 20(R7), V25.W4 |
| VMOVQ 24(R7), V26.W4 |
| VMOVQ 28(R7), V27.W4 |
| |
| // add back the initial state to generate the key stream |
| VADDW V30, V12, V12 // update counter in advance to prevent V30 from being overwritten |
| VADDW V16, V0, V0 |
| VADDW V17, V1, V1 |
| VADDW V18, V2, V2 |
| VADDW V19, V3, V3 |
| |
| // load origin counter + nonce |
| VMOVQ (R9), V28.W4 |
| VMOVQ (R8), V29.W4 |
| VMOVQ 4(R8), V30.W4 |
| VMOVQ 8(R8), V31.W4 |
| |
| VADDW V20, V4, V4 |
| VADDW V21, V5, V5 |
| VADDW V22, V6, V6 |
| VADDW V23, V7, V7 |
| VADDW V24, V8, V8 |
| VADDW V25, V9, V9 |
| VADDW V26, V10, V10 |
| VADDW V27, V11, V11 |
| VADDW V28, V12, V12 |
| VADDW V29, V13, V13 |
| VADDW V30, V14, V14 |
| VADDW V31, V15, V15 |
| |
| // shuffle |
| VILVLW V0, V1, V16 |
| VILVHW V0, V1, V17 |
| VILVLW V2, V3, V18 |
| VILVHW V2, V3, V19 |
| VILVLW V4, V5 ,V20 |
| VILVHW V4, V5, V21 |
| VILVLW V6, V7, V22 |
| VILVHW V6, V7, V23 |
| VILVLW V8, V9, V24 |
| VILVHW V8, V9, V25 |
| VILVLW V10, V11, V26 |
| VILVHW V10, V11, V27 |
| VILVLW V12, V13, V28 |
| VILVHW V12, V13, V29 |
| VILVLW V14, V15, V30 |
| VILVHW V14, V15, V31 |
| VILVLV V16, V18, V0 |
| VILVHV V16, V18, V4 |
| VILVLV V17, V19, V8 |
| VILVHV V17, V19, V12 |
| |
| // load src data from R5 |
| VMOVQ 0(R5), V16 |
| VMOVQ 16(R5), V17 |
| VMOVQ 32(R5), V18 |
| VMOVQ 48(R5), V19 |
| |
| VILVLV V20, V22, V1 |
| VILVHV V20, V22, V5 |
| VILVLV V21, V23, V9 |
| VILVHV V21, V23, V13 |
| |
| VMOVQ 64(R5), V20 |
| VMOVQ 80(R5), V21 |
| VMOVQ 96(R5), V22 |
| VMOVQ 112(R5), V23 |
| |
| VILVLV V24, V26, V2 |
| VILVHV V24, V26, V6 |
| VILVLV V25, V27, V10 |
| VILVHV V25, V27, V14 |
| |
| VMOVQ 128(R5), V24 |
| VMOVQ 144(R5), V25 |
| VMOVQ 160(R5), V26 |
| VMOVQ 176(R5), V27 |
| |
| VILVLV V28, V30, V3 |
| VILVHV V28, V30, V7 |
| VILVLV V29, V31, V11 |
| VILVHV V29, V31, V15 |
| |
| VMOVQ 192(R5), V28 |
| VMOVQ 208(R5), V29 |
| VMOVQ 224(R5), V30 |
| VMOVQ 240(R5), V31 |
| |
| VXORV V0, V16, V16 |
| VXORV V1, V17, V17 |
| VXORV V2, V18, V18 |
| VXORV V3, V19, V19 |
| |
| VMOVQ V16, 0(R4) |
| VMOVQ V17, 16(R4) |
| VMOVQ V18, 32(R4) |
| VMOVQ V19, 48(R4) |
| |
| VXORV V4, V20, V20 |
| VXORV V5, V21, V21 |
| VXORV V6, V22, V22 |
| VXORV V7, V23, V23 |
| |
| VMOVQ V20, 64(R4) |
| VMOVQ V21, 80(R4) |
| VMOVQ V22, 96(R4) |
| VMOVQ V23, 112(R4) |
| |
| VXORV V8, V24, V24 |
| VXORV V9, V25, V25 |
| VXORV V10, V26, V26 |
| VXORV V11, V27, V27 |
| |
| VMOVQ V24, 128(R4) |
| VMOVQ V25, 144(R4) |
| VMOVQ V26, 160(R4) |
| VMOVQ V27, 176(R4) |
| |
| VXORV V12, V28, V28 |
| VXORV V13, V29, V29 |
| VXORV V14, V30, V30 |
| VXORV V15, V31, V31 |
| |
| VMOVQ V28, 192(R4) |
| VMOVQ V29, 208(R4) |
| VMOVQ V30, 224(R4) |
| VMOVQ V31, 240(R4) |
| |
| ADD $4, R12, R12 |
| MOVW R12, (R9) // update counter |
| |
| ADDV $256, R4, R4 |
| ADDV $256, R5, R5 |
| SUBV $256, R6, R6 |
| BNE R6, R0, loop |
| |
| RET |
