| // Copyright 2018 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| //go:build gc && !purego |
| |
| #include "go_asm.h" |
| #include "textflag.h" |
| |
| // This is an implementation of the ChaCha20 encryption algorithm as |
| // specified in RFC 7539. It uses vector instructions to compute |
| // 4 keystream blocks in parallel (256 bytes) which are then XORed |
| // with the bytes in the input slice. |
| |
| GLOBL ·constants<>(SB), RODATA|NOPTR, $32 |
| // BSWAP: swap bytes in each 4-byte element |
| DATA ·constants<>+0x00(SB)/4, $0x03020100 |
| DATA ·constants<>+0x04(SB)/4, $0x07060504 |
| DATA ·constants<>+0x08(SB)/4, $0x0b0a0908 |
| DATA ·constants<>+0x0c(SB)/4, $0x0f0e0d0c |
| // J0: [j0, j1, j2, j3] |
| DATA ·constants<>+0x10(SB)/4, $0x61707865 |
| DATA ·constants<>+0x14(SB)/4, $0x3320646e |
| DATA ·constants<>+0x18(SB)/4, $0x79622d32 |
| DATA ·constants<>+0x1c(SB)/4, $0x6b206574 |
| |
| #define BSWAP V5 |
| #define J0 V6 |
| #define KEY0 V7 |
| #define KEY1 V8 |
| #define NONCE V9 |
| #define CTR V10 |
| #define M0 V11 |
| #define M1 V12 |
| #define M2 V13 |
| #define M3 V14 |
| #define INC V15 |
| #define X0 V16 |
| #define X1 V17 |
| #define X2 V18 |
| #define X3 V19 |
| #define X4 V20 |
| #define X5 V21 |
| #define X6 V22 |
| #define X7 V23 |
| #define X8 V24 |
| #define X9 V25 |
| #define X10 V26 |
| #define X11 V27 |
| #define X12 V28 |
| #define X13 V29 |
| #define X14 V30 |
| #define X15 V31 |
| |
| #define NUM_ROUNDS 20 |
| |
| #define ROUND4(a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3) \ |
| VAF a1, a0, a0 \ |
| VAF b1, b0, b0 \ |
| VAF c1, c0, c0 \ |
| VAF d1, d0, d0 \ |
| VX a0, a2, a2 \ |
| VX b0, b2, b2 \ |
| VX c0, c2, c2 \ |
| VX d0, d2, d2 \ |
| VERLLF $16, a2, a2 \ |
| VERLLF $16, b2, b2 \ |
| VERLLF $16, c2, c2 \ |
| VERLLF $16, d2, d2 \ |
| VAF a2, a3, a3 \ |
| VAF b2, b3, b3 \ |
| VAF c2, c3, c3 \ |
| VAF d2, d3, d3 \ |
| VX a3, a1, a1 \ |
| VX b3, b1, b1 \ |
| VX c3, c1, c1 \ |
| VX d3, d1, d1 \ |
| VERLLF $12, a1, a1 \ |
| VERLLF $12, b1, b1 \ |
| VERLLF $12, c1, c1 \ |
| VERLLF $12, d1, d1 \ |
| VAF a1, a0, a0 \ |
| VAF b1, b0, b0 \ |
| VAF c1, c0, c0 \ |
| VAF d1, d0, d0 \ |
| VX a0, a2, a2 \ |
| VX b0, b2, b2 \ |
| VX c0, c2, c2 \ |
| VX d0, d2, d2 \ |
| VERLLF $8, a2, a2 \ |
| VERLLF $8, b2, b2 \ |
| VERLLF $8, c2, c2 \ |
| VERLLF $8, d2, d2 \ |
| VAF a2, a3, a3 \ |
| VAF b2, b3, b3 \ |
| VAF c2, c3, c3 \ |
| VAF d2, d3, d3 \ |
| VX a3, a1, a1 \ |
| VX b3, b1, b1 \ |
| VX c3, c1, c1 \ |
| VX d3, d1, d1 \ |
| VERLLF $7, a1, a1 \ |
| VERLLF $7, b1, b1 \ |
| VERLLF $7, c1, c1 \ |
| VERLLF $7, d1, d1 |
| |
| #define PERMUTE(mask, v0, v1, v2, v3) \ |
| VPERM v0, v0, mask, v0 \ |
| VPERM v1, v1, mask, v1 \ |
| VPERM v2, v2, mask, v2 \ |
| VPERM v3, v3, mask, v3 |
| |
| #define ADDV(x, v0, v1, v2, v3) \ |
| VAF x, v0, v0 \ |
| VAF x, v1, v1 \ |
| VAF x, v2, v2 \ |
| VAF x, v3, v3 |
| |
| #define XORV(off, dst, src, v0, v1, v2, v3) \ |
| VLM off(src), M0, M3 \ |
| PERMUTE(BSWAP, v0, v1, v2, v3) \ |
| VX v0, M0, M0 \ |
| VX v1, M1, M1 \ |
| VX v2, M2, M2 \ |
| VX v3, M3, M3 \ |
| VSTM M0, M3, off(dst) |
| |
| #define SHUFFLE(a, b, c, d, t, u, v, w) \ |
| VMRHF a, c, t \ // t = {a[0], c[0], a[1], c[1]} |
| VMRHF b, d, u \ // u = {b[0], d[0], b[1], d[1]} |
| VMRLF a, c, v \ // v = {a[2], c[2], a[3], c[3]} |
| VMRLF b, d, w \ // w = {b[2], d[2], b[3], d[3]} |
| VMRHF t, u, a \ // a = {a[0], b[0], c[0], d[0]} |
| VMRLF t, u, b \ // b = {a[1], b[1], c[1], d[1]} |
| VMRHF v, w, c \ // c = {a[2], b[2], c[2], d[2]} |
| VMRLF v, w, d // d = {a[3], b[3], c[3], d[3]} |
| |
| // func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32) |
| TEXT ·xorKeyStreamVX(SB), NOSPLIT, $0 |
| MOVD $·constants<>(SB), R1 |
| MOVD dst+0(FP), R2 // R2=&dst[0] |
| LMG src+24(FP), R3, R4 // R3=&src[0] R4=len(src) |
| MOVD key+48(FP), R5 // R5=key |
| MOVD nonce+56(FP), R6 // R6=nonce |
| MOVD counter+64(FP), R7 // R7=counter |
| |
| // load BSWAP and J0 |
| VLM (R1), BSWAP, J0 |
| |
| // setup |
| MOVD $95, R0 |
| VLM (R5), KEY0, KEY1 |
| VLL R0, (R6), NONCE |
| VZERO M0 |
| VLEIB $7, $32, M0 |
| VSRLB M0, NONCE, NONCE |
| |
| // initialize counter values |
| VLREPF (R7), CTR |
| VZERO INC |
| VLEIF $1, $1, INC |
| VLEIF $2, $2, INC |
| VLEIF $3, $3, INC |
| VAF INC, CTR, CTR |
| VREPIF $4, INC |
| |
| chacha: |
| VREPF $0, J0, X0 |
| VREPF $1, J0, X1 |
| VREPF $2, J0, X2 |
| VREPF $3, J0, X3 |
| VREPF $0, KEY0, X4 |
| VREPF $1, KEY0, X5 |
| VREPF $2, KEY0, X6 |
| VREPF $3, KEY0, X7 |
| VREPF $0, KEY1, X8 |
| VREPF $1, KEY1, X9 |
| VREPF $2, KEY1, X10 |
| VREPF $3, KEY1, X11 |
| VLR CTR, X12 |
| VREPF $1, NONCE, X13 |
| VREPF $2, NONCE, X14 |
| VREPF $3, NONCE, X15 |
| |
| MOVD $(NUM_ROUNDS/2), R1 |
| |
| loop: |
| ROUND4(X0, X4, X12, X8, X1, X5, X13, X9, X2, X6, X14, X10, X3, X7, X15, X11) |
| ROUND4(X0, X5, X15, X10, X1, X6, X12, X11, X2, X7, X13, X8, X3, X4, X14, X9) |
| |
| ADD $-1, R1 |
| BNE loop |
| |
| // decrement length |
| ADD $-256, R4 |
| |
| // rearrange vectors |
| SHUFFLE(X0, X1, X2, X3, M0, M1, M2, M3) |
| ADDV(J0, X0, X1, X2, X3) |
| SHUFFLE(X4, X5, X6, X7, M0, M1, M2, M3) |
| ADDV(KEY0, X4, X5, X6, X7) |
| SHUFFLE(X8, X9, X10, X11, M0, M1, M2, M3) |
| ADDV(KEY1, X8, X9, X10, X11) |
| VAF CTR, X12, X12 |
| SHUFFLE(X12, X13, X14, X15, M0, M1, M2, M3) |
| ADDV(NONCE, X12, X13, X14, X15) |
| |
| // increment counters |
| VAF INC, CTR, CTR |
| |
| // xor keystream with plaintext |
| XORV(0*64, R2, R3, X0, X4, X8, X12) |
| XORV(1*64, R2, R3, X1, X5, X9, X13) |
| XORV(2*64, R2, R3, X2, X6, X10, X14) |
| XORV(3*64, R2, R3, X3, X7, X11, X15) |
| |
| // increment pointers |
| MOVD $256(R2), R2 |
| MOVD $256(R3), R3 |
| |
| CMPBNE R4, $0, chacha |
| |
| VSTEF $0, CTR, (R7) |
| RET |
