| // Copyright 2025 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| // Code generated by 'go generate' (with ./internal/asmgen). DO NOT EDIT. |
| |
| //go:build !math_big_pure_go |
| |
| #include "textflag.h" |
| |
| // func addVV(z, x, y []Word) (c Word) |
| TEXT ·addVV(SB), NOSPLIT, $0 |
| MOVD z_len+8(FP), R0 |
| MOVD x_base+24(FP), R1 |
| MOVD y_base+48(FP), R2 |
| MOVD z_base+0(FP), R3 |
| // compute unrolled loop lengths |
| AND $3, R0, R4 |
| LSR $2, R0 |
| ADDS ZR, R0 // clear carry |
| loop1: |
| CBZ R4, loop1done |
| loop1cont: |
| // unroll 1X |
| MOVD.P 8(R1), R5 |
| MOVD.P 8(R2), R6 |
| ADCS R6, R5 |
| MOVD.P R5, 8(R3) |
| SUB $1, R4 |
| CBNZ R4, loop1cont |
| loop1done: |
| loop4: |
| CBZ R0, loop4done |
| loop4cont: |
| // unroll 4X |
| LDP.P 32(R1), (R4, R5) |
| LDP -16(R1), (R6, R7) |
| LDP.P 32(R2), (R8, R9) |
| LDP -16(R2), (R10, R11) |
| ADCS R8, R4 |
| ADCS R9, R5 |
| ADCS R10, R6 |
| ADCS R11, R7 |
| STP.P (R4, R5), 32(R3) |
| STP (R6, R7), -16(R3) |
| SUB $1, R0 |
| CBNZ R0, loop4cont |
| loop4done: |
| ADC ZR, ZR, R1 // save & convert add carry |
| MOVD R1, c+72(FP) |
| RET |
| |
| // func subVV(z, x, y []Word) (c Word) |
| TEXT ·subVV(SB), NOSPLIT, $0 |
| MOVD z_len+8(FP), R0 |
| MOVD x_base+24(FP), R1 |
| MOVD y_base+48(FP), R2 |
| MOVD z_base+0(FP), R3 |
| // compute unrolled loop lengths |
| AND $3, R0, R4 |
| LSR $2, R0 |
| SUBS ZR, R0 // clear carry |
| loop1: |
| CBZ R4, loop1done |
| loop1cont: |
| // unroll 1X |
| MOVD.P 8(R1), R5 |
| MOVD.P 8(R2), R6 |
| SBCS R6, R5 |
| MOVD.P R5, 8(R3) |
| SUB $1, R4 |
| CBNZ R4, loop1cont |
| loop1done: |
| loop4: |
| CBZ R0, loop4done |
| loop4cont: |
| // unroll 4X |
| LDP.P 32(R1), (R4, R5) |
| LDP -16(R1), (R6, R7) |
| LDP.P 32(R2), (R8, R9) |
| LDP -16(R2), (R10, R11) |
| SBCS R8, R4 |
| SBCS R9, R5 |
| SBCS R10, R6 |
| SBCS R11, R7 |
| STP.P (R4, R5), 32(R3) |
| STP (R6, R7), -16(R3) |
| SUB $1, R0 |
| CBNZ R0, loop4cont |
| loop4done: |
| SBC R1, R1 // save carry |
| SUB R1, ZR, R1 // convert sub carry |
| MOVD R1, c+72(FP) |
| RET |
| |
| // func lshVU(z, x []Word, s uint) (c Word) |
| TEXT ·lshVU(SB), NOSPLIT, $0 |
| MOVD z_len+8(FP), R0 |
| CBZ R0, ret0 |
| MOVD s+48(FP), R1 |
| MOVD x_base+24(FP), R2 |
| MOVD z_base+0(FP), R3 |
| // run loop backward |
| ADD R0<<3, R2, R2 |
| ADD R0<<3, R3, R3 |
| // shift first word into carry |
| MOVD.W -8(R2), R4 |
| MOVD $64, R5 |
| SUB R1, R5 |
| LSR R5, R4, R6 |
| LSL R1, R4 |
| MOVD R6, c+56(FP) |
| // shift remaining words |
| SUB $1, R0 |
| // compute unrolled loop lengths |
| AND $3, R0, R6 |
| LSR $2, R0 |
| loop1: |
| CBZ R6, loop1done |
| loop1cont: |
| // unroll 1X |
| MOVD.W -8(R2), R7 |
| LSR R5, R7, R8 |
| ORR R4, R8 |
| LSL R1, R7, R4 |
| MOVD.W R8, -8(R3) |
| SUB $1, R6 |
| CBNZ R6, loop1cont |
| loop1done: |
| loop4: |
| CBZ R0, loop4done |
| loop4cont: |
| // unroll 4X |
| LDP.W -32(R2), (R9, R8) |
| LDP 16(R2), (R7, R6) |
| LSR R5, R6, R10 |
| ORR R4, R10 |
| LSL R1, R6, R4 |
| LSR R5, R7, R6 |
| ORR R4, R6 |
| LSL R1, R7, R4 |
| LSR R5, R8, R7 |
| ORR R4, R7 |
| LSL R1, R8, R4 |
| LSR R5, R9, R8 |
| ORR R4, R8 |
| LSL R1, R9, R4 |
| STP.W (R8, R7), -32(R3) |
| STP (R6, R10), 16(R3) |
| SUB $1, R0 |
| CBNZ R0, loop4cont |
| loop4done: |
| // store final shifted bits |
| MOVD.W R4, -8(R3) |
| RET |
| ret0: |
| MOVD ZR, c+56(FP) |
| RET |
| |
| // func rshVU(z, x []Word, s uint) (c Word) |
| TEXT ·rshVU(SB), NOSPLIT, $0 |
| MOVD z_len+8(FP), R0 |
| CBZ R0, ret0 |
| MOVD s+48(FP), R1 |
| MOVD x_base+24(FP), R2 |
| MOVD z_base+0(FP), R3 |
| // shift first word into carry |
| MOVD.P 8(R2), R4 |
| MOVD $64, R5 |
| SUB R1, R5 |
| LSL R5, R4, R6 |
| LSR R1, R4 |
| MOVD R6, c+56(FP) |
| // shift remaining words |
| SUB $1, R0 |
| // compute unrolled loop lengths |
| AND $3, R0, R6 |
| LSR $2, R0 |
| loop1: |
| CBZ R6, loop1done |
| loop1cont: |
| // unroll 1X |
| MOVD.P 8(R2), R7 |
| LSL R5, R7, R8 |
| ORR R4, R8 |
| LSR R1, R7, R4 |
| MOVD.P R8, 8(R3) |
| SUB $1, R6 |
| CBNZ R6, loop1cont |
| loop1done: |
| loop4: |
| CBZ R0, loop4done |
| loop4cont: |
| // unroll 4X |
| LDP.P 32(R2), (R6, R7) |
| LDP -16(R2), (R8, R9) |
| LSL R5, R6, R10 |
| ORR R4, R10 |
| LSR R1, R6, R4 |
| LSL R5, R7, R6 |
| ORR R4, R6 |
| LSR R1, R7, R4 |
| LSL R5, R8, R7 |
| ORR R4, R7 |
| LSR R1, R8, R4 |
| LSL R5, R9, R8 |
| ORR R4, R8 |
| LSR R1, R9, R4 |
| STP.P (R10, R6), 32(R3) |
| STP (R7, R8), -16(R3) |
| SUB $1, R0 |
| CBNZ R0, loop4cont |
| loop4done: |
| // store final shifted bits |
| MOVD.P R4, 8(R3) |
| RET |
| ret0: |
| MOVD ZR, c+56(FP) |
| RET |
| |
| // func mulAddVWW(z, x []Word, m, a Word) (c Word) |
| TEXT ·mulAddVWW(SB), NOSPLIT, $0 |
| MOVD m+48(FP), R0 |
| MOVD a+56(FP), R1 |
| MOVD z_len+8(FP), R2 |
| MOVD x_base+24(FP), R3 |
| MOVD z_base+0(FP), R4 |
| // compute unrolled loop lengths |
| AND $7, R2, R5 |
| LSR $3, R2 |
| loop1: |
| CBZ R5, loop1done |
| loop1cont: |
| // unroll 1X |
| MOVD.P 8(R3), R6 |
| // multiply |
| UMULH R0, R6, R7 |
| MUL R0, R6 |
| ADDS R1, R6 |
| ADC ZR, R7, R1 |
| MOVD.P R6, 8(R4) |
| SUB $1, R5 |
| CBNZ R5, loop1cont |
| loop1done: |
| loop8: |
| CBZ R2, loop8done |
| loop8cont: |
| // unroll 8X |
| LDP.P 64(R3), (R5, R6) |
| LDP -48(R3), (R7, R8) |
| LDP -32(R3), (R9, R10) |
| LDP -16(R3), (R11, R12) |
| // multiply |
| UMULH R0, R5, R13 |
| MUL R0, R5 |
| ADDS R1, R5 |
| UMULH R0, R6, R14 |
| MUL R0, R6 |
| ADCS R13, R6 |
| UMULH R0, R7, R13 |
| MUL R0, R7 |
| ADCS R14, R7 |
| UMULH R0, R8, R14 |
| MUL R0, R8 |
| ADCS R13, R8 |
| UMULH R0, R9, R13 |
| MUL R0, R9 |
| ADCS R14, R9 |
| UMULH R0, R10, R14 |
| MUL R0, R10 |
| ADCS R13, R10 |
| UMULH R0, R11, R13 |
| MUL R0, R11 |
| ADCS R14, R11 |
| UMULH R0, R12, R14 |
| MUL R0, R12 |
| ADCS R13, R12 |
| ADC ZR, R14, R1 |
| STP.P (R5, R6), 64(R4) |
| STP (R7, R8), -48(R4) |
| STP (R9, R10), -32(R4) |
| STP (R11, R12), -16(R4) |
| SUB $1, R2 |
| CBNZ R2, loop8cont |
| loop8done: |
| MOVD R1, c+64(FP) |
| RET |
| |
| // func addMulVVWW(z, x, y []Word, m, a Word) (c Word) |
| TEXT ·addMulVVWW(SB), NOSPLIT, $0 |
| MOVD m+72(FP), R0 |
| MOVD a+80(FP), R1 |
| MOVD z_len+8(FP), R2 |
| MOVD x_base+24(FP), R3 |
| MOVD y_base+48(FP), R4 |
| MOVD z_base+0(FP), R5 |
| // compute unrolled loop lengths |
| AND $7, R2, R6 |
| LSR $3, R2 |
| loop1: |
| CBZ R6, loop1done |
| loop1cont: |
| // unroll 1X |
| MOVD.P 8(R3), R7 |
| MOVD.P 8(R4), R8 |
| // multiply |
| UMULH R0, R8, R9 |
| MUL R0, R8 |
| ADDS R1, R8 |
| ADC ZR, R9, R1 |
| // add |
| ADDS R7, R8 |
| ADC ZR, R1 |
| MOVD.P R8, 8(R5) |
| SUB $1, R6 |
| CBNZ R6, loop1cont |
| loop1done: |
| loop8: |
| CBZ R2, loop8done |
| loop8cont: |
| // unroll 8X |
| LDP.P 64(R3), (R6, R7) |
| LDP -48(R3), (R8, R9) |
| LDP -32(R3), (R10, R11) |
| LDP -16(R3), (R12, R13) |
| LDP.P 64(R4), (R14, R15) |
| LDP -48(R4), (R16, R17) |
| LDP -32(R4), (R19, R20) |
| LDP -16(R4), (R21, R22) |
| // multiply |
| UMULH R0, R14, R23 |
| MUL R0, R14 |
| ADDS R1, R14 |
| UMULH R0, R15, R24 |
| MUL R0, R15 |
| ADCS R23, R15 |
| UMULH R0, R16, R23 |
| MUL R0, R16 |
| ADCS R24, R16 |
| UMULH R0, R17, R24 |
| MUL R0, R17 |
| ADCS R23, R17 |
| UMULH R0, R19, R23 |
| MUL R0, R19 |
| ADCS R24, R19 |
| UMULH R0, R20, R24 |
| MUL R0, R20 |
| ADCS R23, R20 |
| UMULH R0, R21, R23 |
| MUL R0, R21 |
| ADCS R24, R21 |
| UMULH R0, R22, R24 |
| MUL R0, R22 |
| ADCS R23, R22 |
| ADC ZR, R24, R1 |
| // add |
| ADDS R6, R14 |
| ADCS R7, R15 |
| ADCS R8, R16 |
| ADCS R9, R17 |
| ADCS R10, R19 |
| ADCS R11, R20 |
| ADCS R12, R21 |
| ADCS R13, R22 |
| ADC ZR, R1 |
| STP.P (R14, R15), 64(R5) |
| STP (R16, R17), -48(R5) |
| STP (R19, R20), -32(R5) |
| STP (R21, R22), -16(R5) |
| SUB $1, R2 |
| CBNZ R2, loop8cont |
| loop8done: |
| MOVD R1, c+88(FP) |
| RET |
