| // Copyright 2013 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| // +build !math_big_pure_go |
| |
| #include "textflag.h" |
| |
| // This file provides fast assembly versions for the elementary |
| // arithmetic operations on vectors implemented in arith.go. |
| |
| // TODO: Consider re-implementing using Advanced SIMD |
| // once the assembler supports those instructions. |
| |
| // func mulWW(x, y Word) (z1, z0 Word) |
| TEXT ·mulWW(SB),NOSPLIT,$0 |
| MOVD x+0(FP), R0 |
| MOVD y+8(FP), R1 |
| MUL R0, R1, R2 |
| UMULH R0, R1, R3 |
| MOVD R3, z1+16(FP) |
| MOVD R2, z0+24(FP) |
| RET |
| |
| |
| // func divWW(x1, x0, y Word) (q, r Word) |
| TEXT ·divWW(SB),NOSPLIT,$0 |
| B ·divWW_g(SB) // ARM64 has no multiword division |
| |
| |
| // func addVV(z, x, y []Word) (c Word) |
| TEXT ·addVV(SB),NOSPLIT,$0 |
| MOVD z_len+8(FP), R0 |
| MOVD x+24(FP), R8 |
| MOVD y+48(FP), R9 |
| MOVD z+0(FP), R10 |
| ADDS $0, R0 // clear carry flag |
| TBZ $0, R0, two |
| MOVD.P 8(R8), R11 |
| MOVD.P 8(R9), R15 |
| ADCS R15, R11 |
| MOVD.P R11, 8(R10) |
| SUB $1, R0 |
| two: |
| TBZ $1, R0, loop |
| LDP.P 16(R8), (R11, R12) |
| LDP.P 16(R9), (R15, R16) |
| ADCS R15, R11 |
| ADCS R16, R12 |
| STP.P (R11, R12), 16(R10) |
| SUB $2, R0 |
| loop: |
| CBZ R0, done // careful not to touch the carry flag |
| LDP.P 32(R8), (R11, R12) |
| LDP -16(R8), (R13, R14) |
| LDP.P 32(R9), (R15, R16) |
| LDP -16(R9), (R17, R19) |
| ADCS R15, R11 |
| ADCS R16, R12 |
| ADCS R17, R13 |
| ADCS R19, R14 |
| STP.P (R11, R12), 32(R10) |
| STP (R13, R14), -16(R10) |
| SUB $4, R0 |
| B loop |
| done: |
| CSET HS, R0 // extract carry flag |
| MOVD R0, c+72(FP) |
| RET |
| |
| |
| // func subVV(z, x, y []Word) (c Word) |
| TEXT ·subVV(SB),NOSPLIT,$0 |
| MOVD z_len+8(FP), R0 |
| MOVD x+24(FP), R8 |
| MOVD y+48(FP), R9 |
| MOVD z+0(FP), R10 |
| CMP R0, R0 // set carry flag |
| TBZ $0, R0, two |
| MOVD.P 8(R8), R11 |
| MOVD.P 8(R9), R15 |
| SBCS R15, R11 |
| MOVD.P R11, 8(R10) |
| SUB $1, R0 |
| two: |
| TBZ $1, R0, loop |
| LDP.P 16(R8), (R11, R12) |
| LDP.P 16(R9), (R15, R16) |
| SBCS R15, R11 |
| SBCS R16, R12 |
| STP.P (R11, R12), 16(R10) |
| SUB $2, R0 |
| loop: |
| CBZ R0, done // careful not to touch the carry flag |
| LDP.P 32(R8), (R11, R12) |
| LDP -16(R8), (R13, R14) |
| LDP.P 32(R9), (R15, R16) |
| LDP -16(R9), (R17, R19) |
| SBCS R15, R11 |
| SBCS R16, R12 |
| SBCS R17, R13 |
| SBCS R19, R14 |
| STP.P (R11, R12), 32(R10) |
| STP (R13, R14), -16(R10) |
| SUB $4, R0 |
| B loop |
| done: |
| CSET LO, R0 // extract carry flag |
| MOVD R0, c+72(FP) |
| RET |
| |
| |
| // func addVW(z, x []Word, y Word) (c Word) |
| TEXT ·addVW(SB),NOSPLIT,$0 |
| MOVD z+0(FP), R3 |
| MOVD z_len+8(FP), R0 |
| MOVD x+24(FP), R1 |
| MOVD y+48(FP), R2 |
| CBZ R0, len0 // the length of z is 0 |
| MOVD.P 8(R1), R4 |
| ADDS R2, R4 // z[0] = x[0] + y, set carry |
| MOVD.P R4, 8(R3) |
| SUB $1, R0 |
| CBZ R0, len1 // the length of z is 1 |
| TBZ $0, R0, two |
| MOVD.P 8(R1), R4 // do it once |
| ADCS $0, R4 |
| MOVD.P R4, 8(R3) |
| SUB $1, R0 |
| two: // do it twice |
| TBZ $1, R0, loop |
| LDP.P 16(R1), (R4, R5) |
| ADCS $0, R4, R8 // c, z[i] = x[i] + c |
| ADCS $0, R5, R9 |
| STP.P (R8, R9), 16(R3) |
| SUB $2, R0 |
| loop: // do four times per round |
| CBZ R0, len1 // careful not to touch the carry flag |
| LDP.P 32(R1), (R4, R5) |
| LDP -16(R1), (R6, R7) |
| ADCS $0, R4, R8 |
| ADCS $0, R5, R9 |
| ADCS $0, R6, R10 |
| ADCS $0, R7, R11 |
| STP.P (R8, R9), 32(R3) |
| STP (R10, R11), -16(R3) |
| SUB $4, R0 |
| B loop |
| len1: |
| CSET HS, R2 // extract carry flag |
| len0: |
| MOVD R2, c+56(FP) |
| RET |
| |
| // func subVW(z, x []Word, y Word) (c Word) |
| TEXT ·subVW(SB),NOSPLIT,$0 |
| MOVD z+0(FP), R3 |
| MOVD z_len+8(FP), R0 |
| MOVD x+24(FP), R1 |
| MOVD y+48(FP), R2 |
| CBZ R0, len0 // the length of z is 0 |
| MOVD.P 8(R1), R4 |
| SUBS R2, R4 // z[0] = x[0] - y, set carry |
| MOVD.P R4, 8(R3) |
| SUB $1, R0 |
| CBZ R0, len1 // the length of z is 1 |
| TBZ $0, R0, two // do it once |
| MOVD.P 8(R1), R4 |
| SBCS $0, R4 |
| MOVD.P R4, 8(R3) |
| SUB $1, R0 |
| two: // do it twice |
| TBZ $1, R0, loop |
| LDP.P 16(R1), (R4, R5) |
| SBCS $0, R4, R8 // c, z[i] = x[i] + c |
| SBCS $0, R5, R9 |
| STP.P (R8, R9), 16(R3) |
| SUB $2, R0 |
| loop: // do four times per round |
| CBZ R0, len1 // careful not to touch the carry flag |
| LDP.P 32(R1), (R4, R5) |
| LDP -16(R1), (R6, R7) |
| SBCS $0, R4, R8 |
| SBCS $0, R5, R9 |
| SBCS $0, R6, R10 |
| SBCS $0, R7, R11 |
| STP.P (R8, R9), 32(R3) |
| STP (R10, R11), -16(R3) |
| SUB $4, R0 |
| B loop |
| len1: |
| CSET LO, R2 // extract carry flag |
| len0: |
| MOVD R2, c+56(FP) |
| RET |
| |
| |
| // func shlVU(z, x []Word, s uint) (c Word) |
| TEXT ·shlVU(SB),NOSPLIT,$0 |
| MOVD z+0(FP), R0 |
| MOVD z_len+8(FP), R1 |
| MOVD x+24(FP), R2 |
| MOVD s+48(FP), R3 |
| MOVD $0, R8 // in order not to affect the first element, R8 is initialized to zero |
| MOVD $64, R4 |
| SUB R3, R4 |
| CBZ R1, len0 |
| CBZ R3, copy // if the number of shift is 0, just copy x to z |
| |
| TBZ $0, R1, two |
| MOVD.P 8(R2), R6 |
| LSR R4, R6, R8 |
| LSL R3, R6 |
| MOVD.P R6, 8(R0) |
| SUB $1, R1 |
| two: |
| TBZ $1, R1, loop |
| LDP.P 16(R2), (R6, R7) |
| LSR R4, R6, R9 |
| LSL R3, R6 |
| ORR R8, R6 |
| LSR R4, R7, R8 |
| LSL R3, R7 |
| ORR R9, R7 |
| STP.P (R6, R7), 16(R0) |
| SUB $2, R1 |
| loop: |
| CBZ R1, done |
| LDP.P 32(R2), (R10, R11) |
| LDP -16(R2), (R12, R13) |
| LSR R4, R10, R20 |
| LSL R3, R10 |
| ORR R8, R10 // z[i] = (x[i] << s) | (x[i-1] >> (64 - s)) |
| LSR R4, R11, R21 |
| LSL R3, R11 |
| ORR R20, R11 |
| LSR R4, R12, R22 |
| LSL R3, R12 |
| ORR R21, R12 |
| LSR R4, R13, R8 |
| LSL R3, R13 |
| ORR R22, R13 |
| STP.P (R10, R11), 32(R0) |
| STP (R12, R13), -16(R0) |
| SUB $4, R1 |
| B loop |
| done: |
| MOVD R8, c+56(FP) // the part moved out from the last element |
| RET |
| copy: |
| TBZ $0, R1, ctwo |
| MOVD.P 8(R2), R3 |
| MOVD.P R3, 8(R0) |
| SUB $1, R1 |
| ctwo: |
| TBZ $1, R1, cloop |
| LDP.P 16(R2), (R4, R5) |
| STP.P (R4, R5), 16(R0) |
| SUB $2, R1 |
| cloop: |
| CBZ R1, len0 |
| LDP.P 32(R2), (R4, R5) |
| LDP -16(R2), (R6, R7) |
| STP.P (R4, R5), 32(R0) |
| STP (R6, R7), -16(R0) |
| SUB $4, R1 |
| B cloop |
| len0: |
| MOVD $0, c+56(FP) |
| RET |
| |
| |
| // func shrVU(z, x []Word, s uint) (c Word) |
| TEXT ·shrVU(SB),NOSPLIT,$0 |
| MOVD z+0(FP), R0 |
| MOVD z_len+8(FP), R1 |
| MOVD x+24(FP), R2 |
| MOVD s+48(FP), R3 |
| MOVD $0, R8 |
| MOVD $64, R4 |
| SUB R3, R4 |
| CBZ R1, len0 |
| CBZ R3, copy // if the number of shift is 0, just copy x to z |
| |
| MOVD.P 8(R2), R20 |
| LSR R3, R20, R8 |
| LSL R4, R20 |
| MOVD R20, c+56(FP) // deal with the first element |
| SUB $1, R1 |
| |
| TBZ $0, R1, two |
| MOVD.P 8(R2), R6 |
| LSL R4, R6, R20 |
| ORR R8, R20 |
| LSR R3, R6, R8 |
| MOVD.P R20, 8(R0) |
| SUB $1, R1 |
| two: |
| TBZ $1, R1, loop |
| LDP.P 16(R2), (R6, R7) |
| LSL R4, R6, R20 |
| LSR R3, R6 |
| ORR R8, R20 |
| LSL R4, R7, R21 |
| LSR R3, R7, R8 |
| ORR R6, R21 |
| STP.P (R20, R21), 16(R0) |
| SUB $2, R1 |
| loop: |
| CBZ R1, done |
| LDP.P 32(R2), (R10, R11) |
| LDP -16(R2), (R12, R13) |
| LSL R4, R10, R20 |
| LSR R3, R10 |
| ORR R8, R20 // z[i] = (x[i] >> s) | (x[i+1] << (64 - s)) |
| LSL R4, R11, R21 |
| LSR R3, R11 |
| ORR R10, R21 |
| LSL R4, R12, R22 |
| LSR R3, R12 |
| ORR R11, R22 |
| LSL R4, R13, R23 |
| LSR R3, R13, R8 |
| ORR R12, R23 |
| STP.P (R20, R21), 32(R0) |
| STP (R22, R23), -16(R0) |
| SUB $4, R1 |
| B loop |
| done: |
| MOVD R8, (R0) // deal with the last element |
| RET |
| copy: |
| TBZ $0, R1, ctwo |
| MOVD.P 8(R2), R3 |
| MOVD.P R3, 8(R0) |
| SUB $1, R1 |
| ctwo: |
| TBZ $1, R1, cloop |
| LDP.P 16(R2), (R4, R5) |
| STP.P (R4, R5), 16(R0) |
| SUB $2, R1 |
| cloop: |
| CBZ R1, len0 |
| LDP.P 32(R2), (R4, R5) |
| LDP -16(R2), (R6, R7) |
| STP.P (R4, R5), 32(R0) |
| STP (R6, R7), -16(R0) |
| SUB $4, R1 |
| B cloop |
| len0: |
| MOVD $0, c+56(FP) |
| RET |
| |
| |
| // func mulAddVWW(z, x []Word, y, r Word) (c Word) |
| TEXT ·mulAddVWW(SB),NOSPLIT,$0 |
| MOVD z+0(FP), R1 |
| MOVD z_len+8(FP), R0 |
| MOVD x+24(FP), R2 |
| MOVD y+48(FP), R3 |
| MOVD r+56(FP), R4 |
| loop: |
| CBZ R0, done |
| MOVD.P 8(R2), R5 |
| UMULH R5, R3, R7 |
| MUL R5, R3, R6 |
| ADDS R4, R6 |
| ADC $0, R7 |
| MOVD.P R6, 8(R1) |
| MOVD R7, R4 |
| SUB $1, R0 |
| B loop |
| done: |
| MOVD R4, c+64(FP) |
| RET |
| |
| |
| // func addMulVVW(z, x []Word, y Word) (c Word) |
| TEXT ·addMulVVW(SB),NOSPLIT,$0 |
| MOVD z+0(FP), R1 |
| MOVD z_len+8(FP), R0 |
| MOVD x+24(FP), R2 |
| MOVD y+48(FP), R3 |
| MOVD $0, R4 |
| |
| TBZ $0, R0, two |
| |
| MOVD.P 8(R2), R5 |
| MOVD (R1), R6 |
| |
| MUL R5, R3, R7 |
| UMULH R5, R3, R8 |
| |
| ADDS R7, R6 |
| ADC $0, R8, R4 |
| |
| MOVD.P R6, 8(R1) |
| SUB $1, R0 |
| |
| two: |
| TBZ $1, R0, loop |
| |
| LDP.P 16(R2), (R5, R10) |
| LDP (R1), (R6, R11) |
| |
| MUL R10, R3, R13 |
| UMULH R10, R3, R12 |
| |
| MUL R5, R3, R7 |
| UMULH R5, R3, R8 |
| |
| ADDS R4, R6 |
| ADCS R13, R11 |
| ADC $0, R12 |
| |
| ADDS R7, R6 |
| ADCS R8, R11 |
| ADC $0, R12, R4 |
| |
| STP.P (R6, R11), 16(R1) |
| SUB $2, R0 |
| |
| // The main loop of this code operates on a block of 4 words every iteration |
| // performing [R4:R12:R11:R10:R9] = R4 + R3 * [R8:R7:R6:R5] + [R12:R11:R10:R9] |
| // where R4 is carried from the previous iteration, R8:R7:R6:R5 hold the next |
| // 4 words of x, R3 is y and R12:R11:R10:R9 are part of the result z. |
| loop: |
| CBZ R0, done |
| |
| LDP.P 16(R2), (R5, R6) |
| LDP.P 16(R2), (R7, R8) |
| |
| LDP (R1), (R9, R10) |
| ADDS R4, R9 |
| MUL R6, R3, R14 |
| ADCS R14, R10 |
| MUL R7, R3, R15 |
| LDP 16(R1), (R11, R12) |
| ADCS R15, R11 |
| MUL R8, R3, R16 |
| ADCS R16, R12 |
| UMULH R8, R3, R20 |
| ADC $0, R20 |
| |
| MUL R5, R3, R13 |
| ADDS R13, R9 |
| UMULH R5, R3, R17 |
| ADCS R17, R10 |
| UMULH R6, R3, R21 |
| STP.P (R9, R10), 16(R1) |
| ADCS R21, R11 |
| UMULH R7, R3, R19 |
| ADCS R19, R12 |
| STP.P (R11, R12), 16(R1) |
| ADC $0, R20, R4 |
| |
| SUB $4, R0 |
| B loop |
| |
| done: |
| MOVD R4, c+56(FP) |
| RET |
| |
| // func divWVW(z []Word, xn Word, x []Word, y Word) (r Word) |
| TEXT ·divWVW(SB),NOSPLIT,$0 |
| B ·divWVW_g(SB) |
