| // Copyright 2019 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| // Based on CRYPTOGAMS code with the following comment: |
| // # ==================================================================== |
| // # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL |
| // # project. The module is, however, dual licensed under OpenSSL and |
| // # CRYPTOGAMS licenses depending on where you obtain it. For further |
| // # details see http://www.openssl.org/~appro/cryptogams/. |
| // # ==================================================================== |
| |
| // Original code can be found at the link below: |
| // https://github.com/dot-asm/cryptogams/commit/a60f5b50ed908e91e5c39ca79126a4a876d5d8ff |
| |
| // There are some differences between CRYPTOGAMS code and this one. The round |
| // loop for "_int" isn't the same as the original. Some adjustments were |
| // necessary because there are less vector registers available. For example, some |
| // X variables (r12, r13, r14, and r15) share the same register used by the |
| // counter. The original code uses ctr to name the counter. Here we use CNT |
| // because golang uses CTR as the counter register name. |
| |
| // +build ppc64le,!gccgo,!appengine |
| |
| #include "textflag.h" |
| |
| #define OUT R3 |
| #define INP R4 |
| #define LEN R5 |
| #define KEY R6 |
| #define CNT R7 |
| |
| #define TEMP R8 |
| |
| #define X0 R11 |
| #define X1 R12 |
| #define X2 R14 |
| #define X3 R15 |
| #define X4 R16 |
| #define X5 R17 |
| #define X6 R18 |
| #define X7 R19 |
| #define X8 R20 |
| #define X9 R21 |
| #define X10 R22 |
| #define X11 R23 |
| #define X12 R24 |
| #define X13 R25 |
| #define X14 R26 |
| #define X15 R27 |
| |
| #define CON0 X0 |
| #define CON1 X1 |
| #define CON2 X2 |
| #define CON3 X3 |
| |
| #define KEY0 X4 |
| #define KEY1 X5 |
| #define KEY2 X6 |
| #define KEY3 X7 |
| #define KEY4 X8 |
| #define KEY5 X9 |
| #define KEY6 X10 |
| #define KEY7 X11 |
| |
| #define CNT0 X12 |
| #define CNT1 X13 |
| #define CNT2 X14 |
| #define CNT3 X15 |
| |
| #define TMP0 R9 |
| #define TMP1 R10 |
| #define TMP2 R28 |
| #define TMP3 R29 |
| |
| #define CONSTS R8 |
| |
| #define A0 V0 |
| #define B0 V1 |
| #define C0 V2 |
| #define D0 V3 |
| #define A1 V4 |
| #define B1 V5 |
| #define C1 V6 |
| #define D1 V7 |
| #define A2 V8 |
| #define B2 V9 |
| #define C2 V10 |
| #define D2 V11 |
| #define T0 V12 |
| #define T1 V13 |
| #define T2 V14 |
| |
| #define K0 V15 |
| #define K1 V16 |
| #define K2 V17 |
| #define K3 V18 |
| #define K4 V19 |
| #define K5 V20 |
| |
| #define FOUR V21 |
| #define SIXTEEN V22 |
| #define TWENTY4 V23 |
| #define TWENTY V24 |
| #define TWELVE V25 |
| #define TWENTY5 V26 |
| #define SEVEN V27 |
| |
| #define INPPERM V28 |
| #define OUTPERM V29 |
| #define OUTMASK V30 |
| |
| #define DD0 V31 |
| #define DD1 SEVEN |
| #define DD2 T0 |
| #define DD3 T1 |
| #define DD4 T2 |
| |
| DATA ·consts+0x00(SB)/8, $0x3320646e61707865 |
| DATA ·consts+0x08(SB)/8, $0x6b20657479622d32 |
| DATA ·consts+0x10(SB)/8, $0x0000000000000001 |
| DATA ·consts+0x18(SB)/8, $0x0000000000000000 |
| DATA ·consts+0x20(SB)/8, $0x0000000000000004 |
| DATA ·consts+0x28(SB)/8, $0x0000000000000000 |
| DATA ·consts+0x30(SB)/8, $0x0a0b08090e0f0c0d |
| DATA ·consts+0x38(SB)/8, $0x0203000106070405 |
| DATA ·consts+0x40(SB)/8, $0x090a0b080d0e0f0c |
| DATA ·consts+0x48(SB)/8, $0x0102030005060704 |
| GLOBL ·consts(SB), RODATA, $80 |
| |
| //func chaCha20_ctr32_vmx(out, inp *byte, len int, key *[32]byte, counter *[16]byte) |
| TEXT ·chaCha20_ctr32_vmx(SB),NOSPLIT|NOFRAME,$0 |
| // Load the arguments inside the registers |
| MOVD out+0(FP), OUT |
| MOVD inp+8(FP), INP |
| MOVD len+16(FP), LEN |
| MOVD key+24(FP), KEY |
| MOVD counter+32(FP), CNT |
| |
| MOVD $·consts(SB), CONSTS // point to consts addr |
| |
| MOVD $16, X0 |
| MOVD $32, X1 |
| MOVD $48, X2 |
| MOVD $64, X3 |
| MOVD $31, X4 |
| MOVD $15, X5 |
| |
| // Load key |
| LVX (KEY)(R0), K1 |
| LVSR (KEY)(R0), T0 |
| LVX (KEY)(X0), K2 |
| LVX (KEY)(X4), DD0 |
| |
| // Load counter |
| LVX (CNT)(R0), K3 |
| LVSR (CNT)(R0), T1 |
| LVX (CNT)(X5), DD1 |
| |
| // Load constants |
| LVX (CONSTS)(R0), K0 |
| LVX (CONSTS)(X0), K5 |
| LVX (CONSTS)(X1), FOUR |
| LVX (CONSTS)(X2), SIXTEEN |
| LVX (CONSTS)(X3), TWENTY4 |
| |
| // Align key and counter |
| VPERM K2, K1, T0, K1 |
| VPERM DD0, K2, T0, K2 |
| VPERM DD1, K3, T1, K3 |
| |
| // Load counter to GPR |
| MOVWZ 0(CNT), CNT0 |
| MOVWZ 4(CNT), CNT1 |
| MOVWZ 8(CNT), CNT2 |
| MOVWZ 12(CNT), CNT3 |
| |
| // Adjust vectors for the initial state |
| VADDUWM K3, K5, K3 |
| VADDUWM K3, K5, K4 |
| VADDUWM K4, K5, K5 |
| |
| // Synthesized constants |
| VSPLTISW $-12, TWENTY |
| VSPLTISW $12, TWELVE |
| VSPLTISW $-7, TWENTY5 |
| |
| VXOR T0, T0, T0 |
| VSPLTISW $-1, OUTMASK |
| LVSR (INP)(R0), INPPERM |
| LVSL (OUT)(R0), OUTPERM |
| VPERM OUTMASK, T0, OUTPERM, OUTMASK |
| |
| loop_outer_vmx: |
| // Load constant |
| MOVD $0x61707865, CON0 |
| MOVD $0x3320646e, CON1 |
| MOVD $0x79622d32, CON2 |
| MOVD $0x6b206574, CON3 |
| |
| VOR K0, K0, A0 |
| VOR K0, K0, A1 |
| VOR K0, K0, A2 |
| VOR K1, K1, B0 |
| |
| MOVD $10, TEMP |
| |
| // Load key to GPR |
| MOVWZ 0(KEY), X4 |
| MOVWZ 4(KEY), X5 |
| MOVWZ 8(KEY), X6 |
| MOVWZ 12(KEY), X7 |
| VOR K1, K1, B1 |
| VOR K1, K1, B2 |
| MOVWZ 16(KEY), X8 |
| MOVWZ 0(CNT), X12 |
| MOVWZ 20(KEY), X9 |
| MOVWZ 4(CNT), X13 |
| VOR K2, K2, C0 |
| VOR K2, K2, C1 |
| MOVWZ 24(KEY), X10 |
| MOVWZ 8(CNT), X14 |
| VOR K2, K2, C2 |
| VOR K3, K3, D0 |
| MOVWZ 28(KEY), X11 |
| MOVWZ 12(CNT), X15 |
| VOR K4, K4, D1 |
| VOR K5, K5, D2 |
| |
| MOVD X4, TMP0 |
| MOVD X5, TMP1 |
| MOVD X6, TMP2 |
| MOVD X7, TMP3 |
| VSPLTISW $7, SEVEN |
| |
| MOVD TEMP, CTR |
| |
| loop_vmx: |
| // CRYPTOGAMS uses a macro to create a loop using perl. This isn't possible |
| // using assembly macros. Therefore, the macro expansion result was used |
| // in order to maintain the algorithm efficiency. |
| // This loop generates three keystream blocks using VMX instructions and, |
| // in parallel, one keystream block using scalar instructions. |
| ADD X4, X0, X0 |
| ADD X5, X1, X1 |
| VADDUWM A0, B0, A0 |
| VADDUWM A1, B1, A1 |
| ADD X6, X2, X2 |
| ADD X7, X3, X3 |
| VADDUWM A2, B2, A2 |
| VXOR D0, A0, D0 |
| XOR X0, X12, X12 |
| XOR X1, X13, X13 |
| VXOR D1, A1, D1 |
| VXOR D2, A2, D2 |
| XOR X2, X14, X14 |
| XOR X3, X15, X15 |
| VPERM D0, D0, SIXTEEN, D0 |
| VPERM D1, D1, SIXTEEN, D1 |
| ROTLW $16, X12, X12 |
| ROTLW $16, X13, X13 |
| VPERM D2, D2, SIXTEEN, D2 |
| VADDUWM C0, D0, C0 |
| ROTLW $16, X14, X14 |
| ROTLW $16, X15, X15 |
| VADDUWM C1, D1, C1 |
| VADDUWM C2, D2, C2 |
| ADD X12, X8, X8 |
| ADD X13, X9, X9 |
| VXOR B0, C0, T0 |
| VXOR B1, C1, T1 |
| ADD X14, X10, X10 |
| ADD X15, X11, X11 |
| VXOR B2, C2, T2 |
| VRLW T0, TWELVE, B0 |
| XOR X8, X4, X4 |
| XOR X9, X5, X5 |
| VRLW T1, TWELVE, B1 |
| VRLW T2, TWELVE, B2 |
| XOR X10, X6, X6 |
| XOR X11, X7, X7 |
| VADDUWM A0, B0, A0 |
| VADDUWM A1, B1, A1 |
| ROTLW $12, X4, X4 |
| ROTLW $12, X5, X5 |
| VADDUWM A2, B2, A2 |
| VXOR D0, A0, D0 |
| ROTLW $12, X6, X6 |
| ROTLW $12, X7, X7 |
| VXOR D1, A1, D1 |
| VXOR D2, A2, D2 |
| ADD X4, X0, X0 |
| ADD X5, X1, X1 |
| VPERM D0, D0, TWENTY4, D0 |
| VPERM D1, D1, TWENTY4, D1 |
| ADD X6, X2, X2 |
| ADD X7, X3, X3 |
| VPERM D2, D2, TWENTY4, D2 |
| VADDUWM C0, D0, C0 |
| XOR X0, X12, X12 |
| XOR X1, X13, X13 |
| VADDUWM C1, D1, C1 |
| VADDUWM C2, D2, C2 |
| XOR X2, X14, X14 |
| XOR X3, X15, X15 |
| VXOR B0, C0, T0 |
| VXOR B1, C1, T1 |
| ROTLW $8, X12, X12 |
| ROTLW $8, X13, X13 |
| VXOR B2, C2, T2 |
| VRLW T0, SEVEN, B0 |
| ROTLW $8, X14, X14 |
| ROTLW $8, X15, X15 |
| VRLW T1, SEVEN, B1 |
| VRLW T2, SEVEN, B2 |
| ADD X12, X8, X8 |
| ADD X13, X9, X9 |
| VSLDOI $8, C0, C0, C0 |
| VSLDOI $8, C1, C1, C1 |
| ADD X14, X10, X10 |
| ADD X15, X11, X11 |
| VSLDOI $8, C2, C2, C2 |
| VSLDOI $12, B0, B0, B0 |
| XOR X8, X4, X4 |
| XOR X9, X5, X5 |
| VSLDOI $12, B1, B1, B1 |
| VSLDOI $12, B2, B2, B2 |
| XOR X10, X6, X6 |
| XOR X11, X7, X7 |
| VSLDOI $4, D0, D0, D0 |
| VSLDOI $4, D1, D1, D1 |
| ROTLW $7, X4, X4 |
| ROTLW $7, X5, X5 |
| VSLDOI $4, D2, D2, D2 |
| VADDUWM A0, B0, A0 |
| ROTLW $7, X6, X6 |
| ROTLW $7, X7, X7 |
| VADDUWM A1, B1, A1 |
| VADDUWM A2, B2, A2 |
| ADD X5, X0, X0 |
| ADD X6, X1, X1 |
| VXOR D0, A0, D0 |
| VXOR D1, A1, D1 |
| ADD X7, X2, X2 |
| ADD X4, X3, X3 |
| VXOR D2, A2, D2 |
| VPERM D0, D0, SIXTEEN, D0 |
| XOR X0, X15, X15 |
| XOR X1, X12, X12 |
| VPERM D1, D1, SIXTEEN, D1 |
| VPERM D2, D2, SIXTEEN, D2 |
| XOR X2, X13, X13 |
| XOR X3, X14, X14 |
| VADDUWM C0, D0, C0 |
| VADDUWM C1, D1, C1 |
| ROTLW $16, X15, X15 |
| ROTLW $16, X12, X12 |
| VADDUWM C2, D2, C2 |
| VXOR B0, C0, T0 |
| ROTLW $16, X13, X13 |
| ROTLW $16, X14, X14 |
| VXOR B1, C1, T1 |
| VXOR B2, C2, T2 |
| ADD X15, X10, X10 |
| ADD X12, X11, X11 |
| VRLW T0, TWELVE, B0 |
| VRLW T1, TWELVE, B1 |
| ADD X13, X8, X8 |
| ADD X14, X9, X9 |
| VRLW T2, TWELVE, B2 |
| VADDUWM A0, B0, A0 |
| XOR X10, X5, X5 |
| XOR X11, X6, X6 |
| VADDUWM A1, B1, A1 |
| VADDUWM A2, B2, A2 |
| XOR X8, X7, X7 |
| XOR X9, X4, X4 |
| VXOR D0, A0, D0 |
| VXOR D1, A1, D1 |
| ROTLW $12, X5, X5 |
| ROTLW $12, X6, X6 |
| VXOR D2, A2, D2 |
| VPERM D0, D0, TWENTY4, D0 |
| ROTLW $12, X7, X7 |
| ROTLW $12, X4, X4 |
| VPERM D1, D1, TWENTY4, D1 |
| VPERM D2, D2, TWENTY4, D2 |
| ADD X5, X0, X0 |
| ADD X6, X1, X1 |
| VADDUWM C0, D0, C0 |
| VADDUWM C1, D1, C1 |
| ADD X7, X2, X2 |
| ADD X4, X3, X3 |
| VADDUWM C2, D2, C2 |
| VXOR B0, C0, T0 |
| XOR X0, X15, X15 |
| XOR X1, X12, X12 |
| VXOR B1, C1, T1 |
| VXOR B2, C2, T2 |
| XOR X2, X13, X13 |
| XOR X3, X14, X14 |
| VRLW T0, SEVEN, B0 |
| VRLW T1, SEVEN, B1 |
| ROTLW $8, X15, X15 |
| ROTLW $8, X12, X12 |
| VRLW T2, SEVEN, B2 |
| VSLDOI $8, C0, C0, C0 |
| ROTLW $8, X13, X13 |
| ROTLW $8, X14, X14 |
| VSLDOI $8, C1, C1, C1 |
| VSLDOI $8, C2, C2, C2 |
| ADD X15, X10, X10 |
| ADD X12, X11, X11 |
| VSLDOI $4, B0, B0, B0 |
| VSLDOI $4, B1, B1, B1 |
| ADD X13, X8, X8 |
| ADD X14, X9, X9 |
| VSLDOI $4, B2, B2, B2 |
| VSLDOI $12, D0, D0, D0 |
| XOR X10, X5, X5 |
| XOR X11, X6, X6 |
| VSLDOI $12, D1, D1, D1 |
| VSLDOI $12, D2, D2, D2 |
| XOR X8, X7, X7 |
| XOR X9, X4, X4 |
| ROTLW $7, X5, X5 |
| ROTLW $7, X6, X6 |
| ROTLW $7, X7, X7 |
| ROTLW $7, X4, X4 |
| BC 0x10, 0, loop_vmx |
| |
| SUB $256, LEN, LEN |
| |
| // Accumulate key block |
| ADD $0x61707865, X0, X0 |
| ADD $0x3320646e, X1, X1 |
| ADD $0x79622d32, X2, X2 |
| ADD $0x6b206574, X3, X3 |
| ADD TMP0, X4, X4 |
| ADD TMP1, X5, X5 |
| ADD TMP2, X6, X6 |
| ADD TMP3, X7, X7 |
| MOVWZ 16(KEY), TMP0 |
| MOVWZ 20(KEY), TMP1 |
| MOVWZ 24(KEY), TMP2 |
| MOVWZ 28(KEY), TMP3 |
| ADD TMP0, X8, X8 |
| ADD TMP1, X9, X9 |
| ADD TMP2, X10, X10 |
| ADD TMP3, X11, X11 |
| |
| MOVWZ 12(CNT), TMP0 |
| MOVWZ 8(CNT), TMP1 |
| MOVWZ 4(CNT), TMP2 |
| MOVWZ 0(CNT), TEMP |
| ADD TMP0, X15, X15 |
| ADD TMP1, X14, X14 |
| ADD TMP2, X13, X13 |
| ADD TEMP, X12, X12 |
| |
| // Accumulate key block |
| VADDUWM A0, K0, A0 |
| VADDUWM A1, K0, A1 |
| VADDUWM A2, K0, A2 |
| VADDUWM B0, K1, B0 |
| VADDUWM B1, K1, B1 |
| VADDUWM B2, K1, B2 |
| VADDUWM C0, K2, C0 |
| VADDUWM C1, K2, C1 |
| VADDUWM C2, K2, C2 |
| VADDUWM D0, K3, D0 |
| VADDUWM D1, K4, D1 |
| VADDUWM D2, K5, D2 |
| |
| // Increment counter |
| ADD $4, TEMP, TEMP |
| MOVW TEMP, 0(CNT) |
| |
| VADDUWM K3, FOUR, K3 |
| VADDUWM K4, FOUR, K4 |
| VADDUWM K5, FOUR, K5 |
| |
| // XOR the input slice (INP) with the keystream, which is stored in GPRs (X0-X3). |
| |
| // Load input (aligned or not) |
| MOVWZ 0(INP), TMP0 |
| MOVWZ 4(INP), TMP1 |
| MOVWZ 8(INP), TMP2 |
| MOVWZ 12(INP), TMP3 |
| |
| // XOR with input |
| XOR TMP0, X0, X0 |
| XOR TMP1, X1, X1 |
| XOR TMP2, X2, X2 |
| XOR TMP3, X3, X3 |
| MOVWZ 16(INP), TMP0 |
| MOVWZ 20(INP), TMP1 |
| MOVWZ 24(INP), TMP2 |
| MOVWZ 28(INP), TMP3 |
| XOR TMP0, X4, X4 |
| XOR TMP1, X5, X5 |
| XOR TMP2, X6, X6 |
| XOR TMP3, X7, X7 |
| MOVWZ 32(INP), TMP0 |
| MOVWZ 36(INP), TMP1 |
| MOVWZ 40(INP), TMP2 |
| MOVWZ 44(INP), TMP3 |
| XOR TMP0, X8, X8 |
| XOR TMP1, X9, X9 |
| XOR TMP2, X10, X10 |
| XOR TMP3, X11, X11 |
| MOVWZ 48(INP), TMP0 |
| MOVWZ 52(INP), TMP1 |
| MOVWZ 56(INP), TMP2 |
| MOVWZ 60(INP), TMP3 |
| XOR TMP0, X12, X12 |
| XOR TMP1, X13, X13 |
| XOR TMP2, X14, X14 |
| XOR TMP3, X15, X15 |
| |
| // Store output (aligned or not) |
| MOVW X0, 0(OUT) |
| MOVW X1, 4(OUT) |
| MOVW X2, 8(OUT) |
| MOVW X3, 12(OUT) |
| |
| ADD $64, INP, INP // INP points to the end of the slice for the alignment code below |
| |
| MOVW X4, 16(OUT) |
| MOVD $16, TMP0 |
| MOVW X5, 20(OUT) |
| MOVD $32, TMP1 |
| MOVW X6, 24(OUT) |
| MOVD $48, TMP2 |
| MOVW X7, 28(OUT) |
| MOVD $64, TMP3 |
| MOVW X8, 32(OUT) |
| MOVW X9, 36(OUT) |
| MOVW X10, 40(OUT) |
| MOVW X11, 44(OUT) |
| MOVW X12, 48(OUT) |
| MOVW X13, 52(OUT) |
| MOVW X14, 56(OUT) |
| MOVW X15, 60(OUT) |
| ADD $64, OUT, OUT |
| |
| // Load input |
| LVX (INP)(R0), DD0 |
| LVX (INP)(TMP0), DD1 |
| LVX (INP)(TMP1), DD2 |
| LVX (INP)(TMP2), DD3 |
| LVX (INP)(TMP3), DD4 |
| ADD $64, INP, INP |
| |
| VPERM DD1, DD0, INPPERM, DD0 // Align input |
| VPERM DD2, DD1, INPPERM, DD1 |
| VPERM DD3, DD2, INPPERM, DD2 |
| VPERM DD4, DD3, INPPERM, DD3 |
| VXOR A0, DD0, A0 // XOR with input |
| VXOR B0, DD1, B0 |
| LVX (INP)(TMP0), DD1 // Keep loading input |
| VXOR C0, DD2, C0 |
| LVX (INP)(TMP1), DD2 |
| VXOR D0, DD3, D0 |
| LVX (INP)(TMP2), DD3 |
| LVX (INP)(TMP3), DD0 |
| ADD $64, INP, INP |
| MOVD $63, TMP3 // 63 is not a typo |
| VPERM A0, A0, OUTPERM, A0 |
| VPERM B0, B0, OUTPERM, B0 |
| VPERM C0, C0, OUTPERM, C0 |
| VPERM D0, D0, OUTPERM, D0 |
| |
| VPERM DD1, DD4, INPPERM, DD4 // Align input |
| VPERM DD2, DD1, INPPERM, DD1 |
| VPERM DD3, DD2, INPPERM, DD2 |
| VPERM DD0, DD3, INPPERM, DD3 |
| VXOR A1, DD4, A1 |
| VXOR B1, DD1, B1 |
| LVX (INP)(TMP0), DD1 // Keep loading |
| VXOR C1, DD2, C1 |
| LVX (INP)(TMP1), DD2 |
| VXOR D1, DD3, D1 |
| LVX (INP)(TMP2), DD3 |
| |
| // Note that the LVX address is always rounded down to the nearest 16-byte |
| // boundary, and that it always points to at most 15 bytes beyond the end of |
| // the slice, so we cannot cross a page boundary. |
| LVX (INP)(TMP3), DD4 // Redundant in aligned case. |
| ADD $64, INP, INP |
| VPERM A1, A1, OUTPERM, A1 // Pre-misalign output |
| VPERM B1, B1, OUTPERM, B1 |
| VPERM C1, C1, OUTPERM, C1 |
| VPERM D1, D1, OUTPERM, D1 |
| |
| VPERM DD1, DD0, INPPERM, DD0 // Align Input |
| VPERM DD2, DD1, INPPERM, DD1 |
| VPERM DD3, DD2, INPPERM, DD2 |
| VPERM DD4, DD3, INPPERM, DD3 |
| VXOR A2, DD0, A2 |
| VXOR B2, DD1, B2 |
| VXOR C2, DD2, C2 |
| VXOR D2, DD3, D2 |
| VPERM A2, A2, OUTPERM, A2 |
| VPERM B2, B2, OUTPERM, B2 |
| VPERM C2, C2, OUTPERM, C2 |
| VPERM D2, D2, OUTPERM, D2 |
| |
| ANDCC $15, OUT, X1 // Is out aligned? |
| MOVD OUT, X0 |
| |
| VSEL A0, B0, OUTMASK, DD0 // Collect pre-misaligned output |
| VSEL B0, C0, OUTMASK, DD1 |
| VSEL C0, D0, OUTMASK, DD2 |
| VSEL D0, A1, OUTMASK, DD3 |
| VSEL A1, B1, OUTMASK, B0 |
| VSEL B1, C1, OUTMASK, C0 |
| VSEL C1, D1, OUTMASK, D0 |
| VSEL D1, A2, OUTMASK, A1 |
| VSEL A2, B2, OUTMASK, B1 |
| VSEL B2, C2, OUTMASK, C1 |
| VSEL C2, D2, OUTMASK, D1 |
| |
| STVX DD0, (OUT+TMP0) |
| STVX DD1, (OUT+TMP1) |
| STVX DD2, (OUT+TMP2) |
| ADD $64, OUT, OUT |
| STVX DD3, (OUT+R0) |
| STVX B0, (OUT+TMP0) |
| STVX C0, (OUT+TMP1) |
| STVX D0, (OUT+TMP2) |
| ADD $64, OUT, OUT |
| STVX A1, (OUT+R0) |
| STVX B1, (OUT+TMP0) |
| STVX C1, (OUT+TMP1) |
| STVX D1, (OUT+TMP2) |
| ADD $64, OUT, OUT |
| |
| BEQ aligned_vmx |
| |
| SUB X1, OUT, X2 // in misaligned case edges |
| MOVD $0, X3 // are written byte-by-byte |
| |
| unaligned_tail_vmx: |
| STVEBX D2, (X2+X3) |
| ADD $1, X3, X3 |
| CMPW X3, X1 |
| BNE unaligned_tail_vmx |
| SUB X1, X0, X2 |
| |
| unaligned_head_vmx: |
| STVEBX A0, (X2+X1) |
| CMPW X1, $15 |
| ADD $1, X1, X1 |
| BNE unaligned_head_vmx |
| |
| CMPU LEN, $255 // done with 256-byte block yet? |
| BGT loop_outer_vmx |
| |
| JMP done_vmx |
| |
| aligned_vmx: |
| STVX A0, (X0+R0) |
| CMPU LEN, $255 // done with 256-byte block yet? |
| BGT loop_outer_vmx |
| |
| done_vmx: |
| RET |
