| // Copyright 2022 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| #include "go_asm.h" |
| #include "textflag.h" |
| |
| // See memmove Go doc for important implementation constraints. |
| |
| // Register map |
| // |
| // to R4 |
| // from R5 |
| // n(aka count) R6 |
| // to-end R7 |
| // from-end R8 |
| // data R11-R18 |
| // tmp R9 |
| |
| // Algorithm: |
| // |
| // Memory alignment check is only performed for copy size greater |
| // than 64 bytes to minimize overhead. |
| // |
| // when copy size <= 64 bytes, jump to label tail, according to the |
| // copy size to select the appropriate case and copy directly. |
| // Based on the common memory access instructions of loong64, the |
| // currently implemented cases are: |
| // move_0, move_1, move_2, move_3, move_4, move_5through7, move_8, |
| // move_9through16, move_17through32, move_33through64 |
| // |
| // when copy size > 64 bytes, use the destination-aligned copying, |
| // adopt the following strategy to copy in 3 parts: |
| // 1. Head: do the memory alignment |
| // 2. Body: a 64-byte loop structure |
| // 3. Tail: processing of the remaining part (<= 64 bytes) |
| // |
| // forward: |
| // |
| // Dst NewDst Dstend |
| // | |<----count after correction---->| |
| // |<-------------count before correction---------->| |
| // |<--8-(Dst&7)-->| |<---64 bytes--->| |
| // +------------------------------------------------+ |
| // | Head | Body | Tail | |
| // +---------------+---------------+----------------+ |
| // NewDst = Dst - (Dst & 7) + 8 |
| // count = count - 8 + (Dst & 7) |
| // Src = Src - (Dst & 7) + 8 |
| // |
| // backward: |
| // |
| // Dst NewDstend Dstend |
| // |<-----count after correction------>| | |
| // |<------------count before correction--------------->| |
| // |<---64 bytes--->| |<---Dstend&7--->| |
| // +----------------------------------------------------+ |
| // | Tail | Body | Head | |
| // +----------------+------------------+----------------+ |
| // NewDstend = Dstend - (Dstend & 7) |
| // count = count - (Dstend & 7) |
| // Srcend = Srcend - (Dstend & 7) |
| |
| // func memmove(to, from unsafe.Pointer, n uintptr) |
| TEXT runtime·memmove<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-24 |
| BEQ R4, R5, move_0 |
| BEQ R6, move_0 |
| |
| ADDV R4, R6, R7 // to-end pointer |
| ADDV R5, R6, R8 // from-end pointer |
| |
| // copy size <= 64 bytes, copy directly, not check aligned |
| tail: |
| // < 2 bytes |
| SGTU $2, R6, R9 |
| BNE R9, move_1 |
| |
| // < 3 bytes |
| SGTU $3, R6, R9 |
| BNE R9, move_2 |
| |
| // < 4 bytes |
| SGTU $4, R6, R9 |
| BNE R9, move_3 |
| |
| // < 5 bytes |
| SGTU $5, R6, R9 |
| BNE R9, move_4 |
| |
| // >= 5 bytes and < 8 bytes |
| SGTU $8, R6, R9 |
| BNE R9, move_5through7 |
| |
| // < 9 bytes |
| SGTU $9, R6, R9 |
| BNE R9, move_8 |
| |
| // >= 9 bytes and < 17 bytes |
| SGTU $17, R6, R9 |
| BNE R9, move_9through16 |
| |
| // >= 17 bytes and < 33 bytes |
| SGTU $33, R6, R9 |
| BNE R9, move_17through32 |
| |
| // >= 33 bytes and < 65 bytes |
| SGTU $65, R6, R9 |
| BNE R9, move_33through64 |
| |
| // >= 65 bytes and < 256 bytes |
| SGTU $256, R6, R9 |
| BNE R9, move_large |
| |
| // >= 256 |
| JMP lasx_move_large |
| |
| move_0: |
| RET |
| |
| move_1: |
| MOVB (R5), R11 |
| MOVB R11, (R4) |
| RET |
| move_2: |
| MOVH (R5), R11 |
| MOVH R11, (R4) |
| RET |
| move_3: |
| MOVH (R5), R11 |
| MOVB -1(R8), R12 |
| MOVH R11, (R4) |
| MOVB R12, -1(R7) |
| RET |
| move_4: |
| MOVW (R5), R11 |
| MOVW R11, (R4) |
| RET |
| move_5through7: |
| MOVW (R5), R11 |
| MOVW -4(R8), R12 |
| MOVW R11, (R4) |
| MOVW R12, -4(R7) |
| RET |
| move_8: |
| MOVV (R5), R11 |
| MOVV R11, (R4) |
| RET |
| move_9through16: |
| MOVV (R5), R11 |
| MOVV -8(R8), R12 |
| MOVV R11, (R4) |
| MOVV R12, -8(R7) |
| RET |
| move_17through32: |
| MOVV (R5), R11 |
| MOVV 8(R5), R12 |
| MOVV -16(R8), R13 |
| MOVV -8(R8), R14 |
| MOVV R11, (R4) |
| MOVV R12, 8(R4) |
| MOVV R13, -16(R7) |
| MOVV R14, -8(R7) |
| RET |
| move_33through64: |
| MOVV (R5), R11 |
| MOVV 8(R5), R12 |
| MOVV 16(R5), R13 |
| MOVV 24(R5), R14 |
| MOVV -32(R8), R15 |
| MOVV -24(R8), R16 |
| MOVV -16(R8), R17 |
| MOVV -8(R8), R18 |
| MOVV R11, (R4) |
| MOVV R12, 8(R4) |
| MOVV R13, 16(R4) |
| MOVV R14, 24(R4) |
| MOVV R15, -32(R7) |
| MOVV R16, -24(R7) |
| MOVV R17, -16(R7) |
| MOVV R18, -8(R7) |
| RET |
| |
| move_large: |
| // if (dst > src) && (dst < (src + count)) |
| // regarded as memory overlap |
| // jump to backward |
| // else |
| // jump to forward |
| BGEU R5, R4, forward |
| ADDV R5, R6, R10 |
| BLTU R4, R10, backward |
| forward: |
| AND $7, R4, R9 // dst & 7 |
| BEQ R9, forward_move_64loop |
| forward_unaligned: |
| MOVV $8, R10 |
| SUBV R9, R10 // head = 8 - (dst & 7) |
| MOVV (R5), R11 |
| SUBV R10, R6 // newcount = count - (8 - (dst & 7)) |
| ADDV R10, R5 // newsrc = src + (8 - (dst & 7)) |
| MOVV (R5), R12 |
| MOVV R11, (R4) |
| ADDV R10, R4 // newdst = dst + (8 - (dst & 7)) |
| MOVV R12, (R4) |
| SUBV $8, R6 |
| ADDV $8, R4 |
| ADDV $8, R5 |
| SGTU $65, R6, R9 |
| BNE R9, move_33through64 |
| forward_move_64loop: |
| SUBV $64, R6 |
| SGTU $64, R6, R9 |
| MOVV (R5), R11 |
| MOVV 8(R5), R12 |
| MOVV 16(R5), R13 |
| MOVV 24(R5), R14 |
| MOVV 32(R5), R15 |
| MOVV 40(R5), R16 |
| MOVV 48(R5), R17 |
| MOVV 56(R5), R18 |
| MOVV R11, (R4) |
| MOVV R12, 8(R4) |
| MOVV R13, 16(R4) |
| MOVV R14, 24(R4) |
| MOVV R15, 32(R4) |
| MOVV R16, 40(R4) |
| MOVV R17, 48(R4) |
| MOVV R18, 56(R4) |
| ADDV $64, R5 |
| ADDV $64, R4 |
| BEQ R9, forward_move_64loop |
| // 0 < remaining_length < 64 |
| BNE R6, tail |
| RET |
| |
| // The backward copy algorithm is the same as the forward |
| // copy, except for the direction. |
| backward: |
| AND $7, R7, R9 // dstend & 7 |
| BEQ R9, backward_move_64loop |
| backward_unaligned: |
| MOVV -8(R8), R11 |
| SUBV R9, R6 // newcount = count - (dstend & 7) |
| SUBV R9, R8 // newsrcend = srcend - (dstend & 7) |
| MOVV -8(R8), R12 |
| MOVV R11, -8(R7) |
| SUBV R9, R7 // newdstend = dstend - (dstend & 7) |
| MOVV R12, -8(R7) |
| SUBV $8, R6 |
| SUBV $8, R7 |
| SUBV $8, R8 |
| SGTU $65, R6, R9 |
| BNE R9, move_33through64 |
| backward_move_64loop: |
| SUBV $64, R6 |
| SGTU $64, R6, R9 |
| MOVV -8(R8), R11 |
| MOVV -16(R8), R12 |
| MOVV -24(R8), R13 |
| MOVV -32(R8), R14 |
| MOVV -40(R8), R15 |
| MOVV -48(R8), R16 |
| MOVV -56(R8), R17 |
| MOVV -64(R8), R18 |
| MOVV R11, -8(R7) |
| MOVV R12, -16(R7) |
| MOVV R13, -24(R7) |
| MOVV R14, -32(R7) |
| MOVV R15, -40(R7) |
| MOVV R16, -48(R7) |
| MOVV R17, -56(R7) |
| MOVV R18, -64(R7) |
| SUBV $64, R7 |
| SUBV $64, R8 |
| BEQ R9, backward_move_64loop |
| // 0 < remaining_length < 64 |
| BNE R6, tail |
| RET |
| |
| // use simd 128 instructions to implement memmove |
| // n >= 256 bytes, check 16-byte alignment |
| lsx_move_large: |
| MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R9 |
| BEQ R9, move_large |
| |
| // if (dst > src) && (dst < (src + count)) |
| // regarded as memory overlap |
| // jump to lsx_backward |
| // else |
| // jump to lsx_forward |
| BGEU R5, R4, lsx_forward |
| ADDV R5, R6, R10 |
| BLTU R4, R10, lsx_backward |
| lsx_forward: |
| AND $15, R4, R9 // dst & 15 |
| BEQ R9, lsx_forward_move_128 |
| lsx_forward_unaligned: |
| MOVV $16, R10 |
| SUBV R9, R10 // head = 16 - (dst & 15) |
| VMOVQ (R5), V0 |
| SUBV R10, R6 // newcount = count - (16 - (dst & 15)) |
| ADDV R10, R5 // newsrc = src + (16 - (dst & 15)) |
| VMOVQ (R5), V1 |
| VMOVQ V0, (R4) |
| ADDV R10, R4 // newdst = dst + (16 - (dst & 15)) |
| VMOVQ V1, (R4) |
| SUBV $16, R6 |
| ADDV $16, R4 |
| ADDV $16, R5 |
| lsx_forward_move_128: |
| SGTU $128, R6, R9 |
| BNE R9, lsx_forward_move_32 |
| lsx_forward_move_128loop: |
| SUBV $128, R6 |
| SGTU $128, R6, R9 |
| VMOVQ 0(R5), V0 |
| VMOVQ 16(R5), V1 |
| VMOVQ 32(R5), V2 |
| VMOVQ 48(R5), V3 |
| VMOVQ 64(R5), V4 |
| VMOVQ 80(R5), V5 |
| VMOVQ 96(R5), V6 |
| VMOVQ 112(R5), V7 |
| VMOVQ V0, 0(R4) |
| VMOVQ V1, 16(R4) |
| VMOVQ V2, 32(R4) |
| VMOVQ V3, 48(R4) |
| VMOVQ V4, 64(R4) |
| VMOVQ V5, 80(R4) |
| VMOVQ V6, 96(R4) |
| VMOVQ V7, 112(R4) |
| ADDV $128, R5 |
| ADDV $128, R4 |
| BEQ R9, lsx_forward_move_128loop |
| lsx_forward_move_32: |
| SGTU $32, R6, R9 |
| BNE R9, lsx_forward_move_tail |
| lsx_forward_move_32loop: |
| SUBV $32, R6 |
| SGTU $32, R6, R9 |
| VMOVQ 0(R5), V0 |
| VMOVQ 16(R5), V1 |
| VMOVQ V0, 0(R4) |
| VMOVQ V1, 16(R4) |
| ADDV $32, R5 |
| ADDV $32, R4 |
| BEQ R9, lsx_forward_move_32loop |
| lsx_forward_move_tail: |
| // 0 < remaining_length < 64 |
| BNE R6, tail |
| RET |
| |
| lsx_backward: |
| AND $15, R7, R9 // dstend & 15 |
| BEQ R9, lsx_backward_move_128 |
| lsx_backward_unaligned: |
| VMOVQ -16(R8), V0 |
| SUBV R9, R6 // newcount = count - (dstend & 15) |
| SUBV R9, R8 // newsrcend = srcend - (dstend & 15) |
| VMOVQ -16(R8), V1 |
| VMOVQ V0, -16(R7) |
| SUBV R9, R7 // newdstend = dstend - (dstend & 15) |
| VMOVQ V1, -16(R7) |
| SUBV $16, R6 |
| SUBV $16, R7 |
| SUBV $16, R8 |
| lsx_backward_move_128: |
| SGTU $128, R6, R9 |
| BNE R9, lsx_backward_move_32 |
| lsx_backward_move_128loop: |
| SUBV $128, R6 |
| SGTU $128, R6, R9 |
| VMOVQ -16(R8), V0 |
| VMOVQ -32(R8), V1 |
| VMOVQ -48(R8), V2 |
| VMOVQ -64(R8), V3 |
| VMOVQ -80(R8), V4 |
| VMOVQ -96(R8), V5 |
| VMOVQ -112(R8), V6 |
| VMOVQ -128(R8), V7 |
| VMOVQ V0, -16(R7) |
| VMOVQ V1, -32(R7) |
| VMOVQ V2, -48(R7) |
| VMOVQ V3, -64(R7) |
| VMOVQ V4, -80(R7) |
| VMOVQ V5, -96(R7) |
| VMOVQ V6, -112(R7) |
| VMOVQ V7, -128(R7) |
| SUBV $128, R8 |
| SUBV $128, R7 |
| BEQ R9, lsx_backward_move_128loop |
| lsx_backward_move_32: |
| SGTU $32, R6, R9 |
| BNE R9, lsx_backward_move_tail |
| lsx_backward_move_32loop: |
| SUBV $32, R6 |
| SGTU $32, R6, R9 |
| VMOVQ -16(R8), V0 |
| VMOVQ -32(R8), V1 |
| VMOVQ V0, -16(R7) |
| VMOVQ V1, -32(R7) |
| SUBV $32, R8 |
| SUBV $32, R7 |
| BEQ R9, lsx_backward_move_32loop |
| lsx_backward_move_tail: |
| // 0 < remaining_length < 64 |
| BNE R6, tail |
| RET |
| |
| // use simd 256 instructions to implement memmove |
| // n >= 256 bytes, check 32-byte alignment |
| lasx_move_large: |
| MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLASX(SB), R9 |
| BEQ R9, lsx_move_large |
| |
| // if (dst > src) && (dst < (src + count)) |
| // regarded as memory overlap |
| // jump to lasx_backward |
| // else |
| // jump to lasx_forward |
| BGEU R5, R4, lasx_forward |
| ADDV R5, R6, R10 |
| BLTU R4, R10, lasx_backward |
| lasx_forward: |
| AND $31, R4, R9 // dst & 31 |
| BEQ R9, lasx_forward_move_256 |
| lasx_forward_unaligned: |
| MOVV $32, R10 |
| SUBV R9, R10 // head = 32 - (dst & 31) |
| XVMOVQ (R5), X0 |
| SUBV R10, R6 // newcount = count - (32 - (dst & 31)) |
| ADDV R10, R5 // newsrc = src + (32 - (dst & 31)) |
| XVMOVQ (R5), X1 |
| XVMOVQ X0, (R4) |
| ADDV R10, R4 // newdst = dst + (32 - (dst & 31)) |
| XVMOVQ X1, (R4) |
| SUBV $32, R6 |
| ADDV $32, R4 |
| ADDV $32, R5 |
| lasx_forward_move_256: |
| SGTU $256, R6, R9 |
| BNE R9, lasx_forward_move_64 |
| lasx_forward_move_256loop: |
| SUBV $256, R6 |
| SGTU $256, R6, R9 |
| XVMOVQ 0(R5), X0 |
| XVMOVQ 32(R5), X1 |
| XVMOVQ 64(R5), X2 |
| XVMOVQ 96(R5), X3 |
| XVMOVQ 128(R5), X4 |
| XVMOVQ 160(R5), X5 |
| XVMOVQ 192(R5), X6 |
| XVMOVQ 224(R5), X7 |
| XVMOVQ X0, 0(R4) |
| XVMOVQ X1, 32(R4) |
| XVMOVQ X2, 64(R4) |
| XVMOVQ X3, 96(R4) |
| XVMOVQ X4, 128(R4) |
| XVMOVQ X5, 160(R4) |
| XVMOVQ X6, 192(R4) |
| XVMOVQ X7, 224(R4) |
| ADDV $256, R5 |
| ADDV $256, R4 |
| BEQ R9, lasx_forward_move_256loop |
| lasx_forward_move_64: |
| SGTU $64, R6, R9 |
| BNE R9, lasx_forward_move_tail |
| lasx_forward_move_64loop: |
| SUBV $64, R6 |
| SGTU $64, R6, R9 |
| XVMOVQ (R5), X0 |
| XVMOVQ 32(R5), X1 |
| XVMOVQ X0, (R4) |
| XVMOVQ X1, 32(R4) |
| ADDV $64, R5 |
| ADDV $64, R4 |
| BEQ R9, lasx_forward_move_64loop |
| lasx_forward_move_tail: |
| // 0 < remaining_length < 64 |
| BNE R6, tail |
| RET |
| |
| lasx_backward: |
| AND $31, R7, R9 // dstend & 31 |
| BEQ R9, lasx_backward_move_256 |
| lasx_backward_unaligned: |
| XVMOVQ -32(R8), X0 |
| SUBV R9, R6 // newcount = count - (dstend & 31) |
| SUBV R9, R8 // newsrcend = srcend - (dstend & 31) |
| XVMOVQ -32(R8), X1 |
| XVMOVQ X0, -32(R7) |
| SUBV R9, R7 // newdstend = dstend - (dstend & 31) |
| XVMOVQ X1, -32(R7) |
| SUBV $32, R6 |
| SUBV $32, R7 |
| SUBV $32, R8 |
| lasx_backward_move_256: |
| SGTU $256, R6, R9 |
| BNE R9, lasx_backward_move_64 |
| lasx_backward_move_256loop: |
| SUBV $256, R6 |
| SGTU $256, R6, R9 |
| XVMOVQ -32(R8), X0 |
| XVMOVQ -64(R8), X1 |
| XVMOVQ -96(R8), X2 |
| XVMOVQ -128(R8), X3 |
| XVMOVQ -160(R8), X4 |
| XVMOVQ -192(R8), X5 |
| XVMOVQ -224(R8), X6 |
| XVMOVQ -256(R8), X7 |
| XVMOVQ X0, -32(R7) |
| XVMOVQ X1, -64(R7) |
| XVMOVQ X2, -96(R7) |
| XVMOVQ X3, -128(R7) |
| XVMOVQ X4, -160(R7) |
| XVMOVQ X5, -192(R7) |
| XVMOVQ X6, -224(R7) |
| XVMOVQ X7, -256(R7) |
| SUBV $256, R8 |
| SUBV $256, R7 |
| BEQ R9, lasx_backward_move_256loop |
| lasx_backward_move_64: |
| SGTU $64, R6, R9 |
| BNE R9, lasx_backward_move_tail |
| lasx_backward_move_64loop: |
| SUBV $64, R6 |
| SGTU $64, R6, R9 |
| XVMOVQ -32(R8), X0 |
| XVMOVQ -64(R8), X1 |
| XVMOVQ X0, -32(R7) |
| XVMOVQ X1, -64(R7) |
| SUBV $64, R8 |
| SUBV $64, R7 |
| BEQ R9, lasx_backward_move_64loop |
| lasx_backward_move_tail: |
| // 0 < remaining_length < 64 |
| BNE R6, tail |
| RET |
