| // Copyright 2022 The Go Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style | |
| // license that can be found in the LICENSE file. | |
| #include "go_asm.h" | |
| #include "textflag.h" | |
| // See memmove Go doc for important implementation constraints. | |
| // Register map | |
| // | |
| // to R4 | |
| // from R5 | |
| // n(aka count) R6 | |
| // to-end R7 | |
| // from-end R8 | |
| // data R11-R18 | |
| // tmp R9 | |
| // Algorithm: | |
| // | |
| // Memory alignment check is only performed for copy size greater | |
| // than 64 bytes to minimize overhead. | |
| // | |
| // when copy size <= 64 bytes, jump to label tail, according to the | |
| // copy size to select the appropriate case and copy directly. | |
| // Based on the common memory access instructions of loong64, the | |
| // currently implemented cases are: | |
| // move_0, move_1, move_2, move_3, move_4, move_5through7, move_8, | |
| // move_9through16, move_17through32, move_33through64 | |
| // | |
| // when copy size > 64 bytes, use the destination-aligned copying, | |
| // adopt the following strategy to copy in 3 parts: | |
| // 1. Head: do the memory alignment | |
| // 2. Body: a 64-byte loop structure | |
| // 3. Tail: processing of the remaining part (<= 64 bytes) | |
| // | |
| // forward: | |
| // | |
| // Dst NewDst Dstend | |
| // | |<----count after correction---->| | |
| // |<-------------count before correction---------->| | |
| // |<--8-(Dst&7)-->| |<---64 bytes--->| | |
| // +------------------------------------------------+ | |
| // | Head | Body | Tail | | |
| // +---------------+---------------+----------------+ | |
| // NewDst = Dst - (Dst & 7) + 8 | |
| // count = count - 8 + (Dst & 7) | |
| // Src = Src - (Dst & 7) + 8 | |
| // | |
| // backward: | |
| // | |
| // Dst NewDstend Dstend | |
| // |<-----count after correction------>| | | |
| // |<------------count before correction--------------->| | |
| // |<---64 bytes--->| |<---Dstend&7--->| | |
| // +----------------------------------------------------+ | |
| // | Tail | Body | Head | | |
| // +----------------+------------------+----------------+ | |
| // NewDstend = Dstend - (Dstend & 7) | |
| // count = count - (Dstend & 7) | |
| // Srcend = Srcend - (Dstend & 7) | |
| // func memmove(to, from unsafe.Pointer, n uintptr) | |
| TEXT runtime·memmove<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-24 | |
| BEQ R4, R5, move_0 | |
| BEQ R6, move_0 | |
| ADDV R4, R6, R7 // to-end pointer | |
| ADDV R5, R6, R8 // from-end pointer | |
| // copy size <= 64 bytes, copy directly, not check aligned | |
| tail: | |
| // < 2 bytes | |
| SGTU $2, R6, R9 | |
| BNE R9, move_1 | |
| // < 3 bytes | |
| SGTU $3, R6, R9 | |
| BNE R9, move_2 | |
| // < 4 bytes | |
| SGTU $4, R6, R9 | |
| BNE R9, move_3 | |
| // < 5 bytes | |
| SGTU $5, R6, R9 | |
| BNE R9, move_4 | |
| // >= 5 bytes and < 8 bytes | |
| SGTU $8, R6, R9 | |
| BNE R9, move_5through7 | |
| // < 9 bytes | |
| SGTU $9, R6, R9 | |
| BNE R9, move_8 | |
| // >= 9 bytes and < 17 bytes | |
| SGTU $17, R6, R9 | |
| BNE R9, move_9through16 | |
| // >= 17 bytes and < 33 bytes | |
| SGTU $33, R6, R9 | |
| BNE R9, move_17through32 | |
| // >= 33 bytes and < 65 bytes: dispatch to scalar or vector | |
| // (LSX) handling depending on dst/src alignment. | |
| SGTU $65, R6, R9 | |
| BNE R9, move_33through64_dispatch | |
| // >= 65 bytes and < 256 bytes | |
| SGTU $256, R6, R9 | |
| BNE R9, move_large | |
| // >= 256 | |
| JMP lasx_move_large | |
| move_0: | |
| RET | |
| move_1: | |
| MOVB (R5), R11 | |
| MOVB R11, (R4) | |
| RET | |
| move_2: | |
| MOVH (R5), R11 | |
| MOVH R11, (R4) | |
| RET | |
| move_3: | |
| MOVH (R5), R11 | |
| MOVB -1(R8), R12 | |
| MOVH R11, (R4) | |
| MOVB R12, -1(R7) | |
| RET | |
| move_4: | |
| MOVW (R5), R11 | |
| MOVW R11, (R4) | |
| RET | |
| move_5through7: | |
| MOVW (R5), R11 | |
| MOVW -4(R8), R12 | |
| MOVW R11, (R4) | |
| MOVW R12, -4(R7) | |
| RET | |
| move_8: | |
| MOVV (R5), R11 | |
| MOVV R11, (R4) | |
| RET | |
| move_9through16: | |
| MOVV (R5), R11 | |
| MOVV -8(R8), R12 | |
| MOVV R11, (R4) | |
| MOVV R12, -8(R7) | |
| RET | |
| move_17through32: | |
| MOVV (R5), R11 | |
| MOVV 8(R5), R12 | |
| MOVV -16(R8), R13 | |
| MOVV -8(R8), R14 | |
| MOVV R11, (R4) | |
| MOVV R12, 8(R4) | |
| MOVV R13, -16(R7) | |
| MOVV R14, -8(R7) | |
| RET | |
| move_33through64: | |
| MOVV (R5), R11 | |
| MOVV 8(R5), R12 | |
| MOVV 16(R5), R13 | |
| MOVV 24(R5), R14 | |
| MOVV -32(R8), R15 | |
| MOVV -24(R8), R16 | |
| MOVV -16(R8), R17 | |
| MOVV -8(R8), R18 | |
| MOVV R11, (R4) | |
| MOVV R12, 8(R4) | |
| MOVV R13, 16(R4) | |
| MOVV R14, 24(R4) | |
| MOVV R15, -32(R7) | |
| MOVV R16, -24(R7) | |
| MOVV R17, -16(R7) | |
| MOVV R18, -8(R7) | |
| RET | |
| // move_33through64_dispatch only runs for the 33~64 byte bucket. | |
| // Aligned dst/src (the common case) falls straight through to the | |
| // original scalar move_33through64 after 3 extra instructions | |
| // (OR/AND/BEQ). Unaligned dst/src without LSX support falls through | |
| // after 5 extra instructions (OR/AND/BEQ/MOVBU/BEQ). Unaligned | |
| // dst/src with LSX available uses four 128-bit vector load/store | |
| // pairs instead of the eight scalar MOVV load/store pairs used by | |
| // move_33through64. | |
| move_33through64_dispatch: | |
| OR R4, R5, R9 | |
| AND $7, R9 | |
| BEQ R9, move_33through64 | |
| MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R9 | |
| BEQ R9, move_33through64 | |
| JMP lsx_move_33through64 | |
| lsx_move_33through64: | |
| VMOVQ (R5), V0 | |
| VMOVQ 16(R5), V1 | |
| VMOVQ -32(R8), V2 | |
| VMOVQ -16(R8), V3 | |
| VMOVQ V0, (R4) | |
| VMOVQ V1, 16(R4) | |
| VMOVQ V2, -32(R7) | |
| VMOVQ V3, -16(R7) | |
| RET | |
| move_large: | |
| // if (dst > src) && (dst < (src + count)) | |
| // regarded as memory overlap | |
| // jump to backward | |
| // else | |
| // jump to forward | |
| BGEU R5, R4, forward | |
| ADDV R5, R6, R10 | |
| BLTU R4, R10, backward | |
| forward: | |
| AND $7, R4, R9 // dst & 7 | |
| BEQ R9, forward_move_64loop | |
| forward_unaligned: | |
| MOVV $8, R10 | |
| SUBV R9, R10 // head = 8 - (dst & 7) | |
| MOVV (R5), R11 | |
| SUBV R10, R6 // newcount = count - (8 - (dst & 7)) | |
| ADDV R10, R5 // newsrc = src + (8 - (dst & 7)) | |
| MOVV (R5), R12 | |
| MOVV R11, (R4) | |
| ADDV R10, R4 // newdst = dst + (8 - (dst & 7)) | |
| MOVV R12, (R4) | |
| SUBV $8, R6 | |
| ADDV $8, R4 | |
| ADDV $8, R5 | |
| SGTU $65, R6, R9 | |
| BNE R9, move_33through64 | |
| forward_move_64loop: | |
| SUBV $64, R6 | |
| SGTU $64, R6, R9 | |
| MOVV (R5), R11 | |
| MOVV 8(R5), R12 | |
| MOVV 16(R5), R13 | |
| MOVV 24(R5), R14 | |
| MOVV 32(R5), R15 | |
| MOVV 40(R5), R16 | |
| MOVV 48(R5), R17 | |
| MOVV 56(R5), R18 | |
| MOVV R11, (R4) | |
| MOVV R12, 8(R4) | |
| MOVV R13, 16(R4) | |
| MOVV R14, 24(R4) | |
| MOVV R15, 32(R4) | |
| MOVV R16, 40(R4) | |
| MOVV R17, 48(R4) | |
| MOVV R18, 56(R4) | |
| ADDV $64, R5 | |
| ADDV $64, R4 | |
| BEQ R9, forward_move_64loop | |
| // 0 < remaining_length < 64 | |
| BNE R6, tail | |
| RET | |
| // The backward copy algorithm is the same as the forward | |
| // copy, except for the direction. | |
| backward: | |
| AND $7, R7, R9 // dstend & 7 | |
| BEQ R9, backward_move_64loop | |
| backward_unaligned: | |
| MOVV -8(R8), R11 | |
| SUBV R9, R6 // newcount = count - (dstend & 7) | |
| SUBV R9, R8 // newsrcend = srcend - (dstend & 7) | |
| MOVV -8(R8), R12 | |
| MOVV R11, -8(R7) | |
| SUBV R9, R7 // newdstend = dstend - (dstend & 7) | |
| MOVV R12, -8(R7) | |
| SUBV $8, R6 | |
| SUBV $8, R7 | |
| SUBV $8, R8 | |
| SGTU $65, R6, R9 | |
| BNE R9, move_33through64 | |
| backward_move_64loop: | |
| SUBV $64, R6 | |
| SGTU $64, R6, R9 | |
| MOVV -8(R8), R11 | |
| MOVV -16(R8), R12 | |
| MOVV -24(R8), R13 | |
| MOVV -32(R8), R14 | |
| MOVV -40(R8), R15 | |
| MOVV -48(R8), R16 | |
| MOVV -56(R8), R17 | |
| MOVV -64(R8), R18 | |
| MOVV R11, -8(R7) | |
| MOVV R12, -16(R7) | |
| MOVV R13, -24(R7) | |
| MOVV R14, -32(R7) | |
| MOVV R15, -40(R7) | |
| MOVV R16, -48(R7) | |
| MOVV R17, -56(R7) | |
| MOVV R18, -64(R7) | |
| SUBV $64, R7 | |
| SUBV $64, R8 | |
| BEQ R9, backward_move_64loop | |
| // 0 < remaining_length < 64 | |
| BNE R6, tail | |
| RET | |
| // use simd 128 instructions to implement memmove | |
| // n >= 256 bytes, check 16-byte alignment | |
| lsx_move_large: | |
| MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R9 | |
| BEQ R9, move_large | |
| // if (dst > src) && (dst < (src + count)) | |
| // regarded as memory overlap | |
| // jump to lsx_backward | |
| // else | |
| // jump to lsx_forward | |
| BGEU R5, R4, lsx_forward | |
| ADDV R5, R6, R10 | |
| BLTU R4, R10, lsx_backward | |
| lsx_forward: | |
| AND $15, R4, R9 // dst & 15 | |
| BEQ R9, lsx_forward_move_128 | |
| lsx_forward_unaligned: | |
| MOVV $16, R10 | |
| SUBV R9, R10 // head = 16 - (dst & 15) | |
| VMOVQ (R5), V0 | |
| SUBV R10, R6 // newcount = count - (16 - (dst & 15)) | |
| ADDV R10, R5 // newsrc = src + (16 - (dst & 15)) | |
| VMOVQ (R5), V1 | |
| VMOVQ V0, (R4) | |
| ADDV R10, R4 // newdst = dst + (16 - (dst & 15)) | |
| VMOVQ V1, (R4) | |
| SUBV $16, R6 | |
| ADDV $16, R4 | |
| ADDV $16, R5 | |
| lsx_forward_move_128: | |
| SGTU $128, R6, R9 | |
| BNE R9, lsx_forward_move_32 | |
| lsx_forward_move_128loop: | |
| SUBV $128, R6 | |
| SGTU $128, R6, R9 | |
| VMOVQ 0(R5), V0 | |
| VMOVQ 16(R5), V1 | |
| VMOVQ 32(R5), V2 | |
| VMOVQ 48(R5), V3 | |
| VMOVQ 64(R5), V4 | |
| VMOVQ 80(R5), V5 | |
| VMOVQ 96(R5), V6 | |
| VMOVQ 112(R5), V7 | |
| VMOVQ V0, 0(R4) | |
| VMOVQ V1, 16(R4) | |
| VMOVQ V2, 32(R4) | |
| VMOVQ V3, 48(R4) | |
| VMOVQ V4, 64(R4) | |
| VMOVQ V5, 80(R4) | |
| VMOVQ V6, 96(R4) | |
| VMOVQ V7, 112(R4) | |
| ADDV $128, R5 | |
| ADDV $128, R4 | |
| BEQ R9, lsx_forward_move_128loop | |
| lsx_forward_move_32: | |
| SGTU $32, R6, R9 | |
| BNE R9, lsx_forward_move_tail | |
| lsx_forward_move_32loop: | |
| SUBV $32, R6 | |
| SGTU $32, R6, R9 | |
| VMOVQ 0(R5), V0 | |
| VMOVQ 16(R5), V1 | |
| VMOVQ V0, 0(R4) | |
| VMOVQ V1, 16(R4) | |
| ADDV $32, R5 | |
| ADDV $32, R4 | |
| BEQ R9, lsx_forward_move_32loop | |
| lsx_forward_move_tail: | |
| // 0 < remaining_length < 64 | |
| BNE R6, tail | |
| RET | |
| lsx_backward: | |
| AND $15, R7, R9 // dstend & 15 | |
| BEQ R9, lsx_backward_move_128 | |
| lsx_backward_unaligned: | |
| VMOVQ -16(R8), V0 | |
| SUBV R9, R6 // newcount = count - (dstend & 15) | |
| SUBV R9, R8 // newsrcend = srcend - (dstend & 15) | |
| VMOVQ -16(R8), V1 | |
| VMOVQ V0, -16(R7) | |
| SUBV R9, R7 // newdstend = dstend - (dstend & 15) | |
| VMOVQ V1, -16(R7) | |
| SUBV $16, R6 | |
| SUBV $16, R7 | |
| SUBV $16, R8 | |
| lsx_backward_move_128: | |
| SGTU $128, R6, R9 | |
| BNE R9, lsx_backward_move_32 | |
| lsx_backward_move_128loop: | |
| SUBV $128, R6 | |
| SGTU $128, R6, R9 | |
| VMOVQ -16(R8), V0 | |
| VMOVQ -32(R8), V1 | |
| VMOVQ -48(R8), V2 | |
| VMOVQ -64(R8), V3 | |
| VMOVQ -80(R8), V4 | |
| VMOVQ -96(R8), V5 | |
| VMOVQ -112(R8), V6 | |
| VMOVQ -128(R8), V7 | |
| VMOVQ V0, -16(R7) | |
| VMOVQ V1, -32(R7) | |
| VMOVQ V2, -48(R7) | |
| VMOVQ V3, -64(R7) | |
| VMOVQ V4, -80(R7) | |
| VMOVQ V5, -96(R7) | |
| VMOVQ V6, -112(R7) | |
| VMOVQ V7, -128(R7) | |
| SUBV $128, R8 | |
| SUBV $128, R7 | |
| BEQ R9, lsx_backward_move_128loop | |
| lsx_backward_move_32: | |
| SGTU $32, R6, R9 | |
| BNE R9, lsx_backward_move_tail | |
| lsx_backward_move_32loop: | |
| SUBV $32, R6 | |
| SGTU $32, R6, R9 | |
| VMOVQ -16(R8), V0 | |
| VMOVQ -32(R8), V1 | |
| VMOVQ V0, -16(R7) | |
| VMOVQ V1, -32(R7) | |
| SUBV $32, R8 | |
| SUBV $32, R7 | |
| BEQ R9, lsx_backward_move_32loop | |
| lsx_backward_move_tail: | |
| // 0 < remaining_length < 64 | |
| BNE R6, tail | |
| RET | |
| // use simd 256 instructions to implement memmove | |
| // n >= 256 bytes, check 32-byte alignment | |
| lasx_move_large: | |
| MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLASX(SB), R9 | |
| BEQ R9, lsx_move_large | |
| // if (dst > src) && (dst < (src + count)) | |
| // regarded as memory overlap | |
| // jump to lasx_backward | |
| // else | |
| // jump to lasx_forward | |
| BGEU R5, R4, lasx_forward | |
| ADDV R5, R6, R10 | |
| BLTU R4, R10, lasx_backward | |
| lasx_forward: | |
| AND $31, R4, R9 // dst & 31 | |
| BEQ R9, lasx_forward_move_256 | |
| lasx_forward_unaligned: | |
| MOVV $32, R10 | |
| SUBV R9, R10 // head = 32 - (dst & 31) | |
| XVMOVQ (R5), X0 | |
| SUBV R10, R6 // newcount = count - (32 - (dst & 31)) | |
| ADDV R10, R5 // newsrc = src + (32 - (dst & 31)) | |
| XVMOVQ (R5), X1 | |
| XVMOVQ X0, (R4) | |
| ADDV R10, R4 // newdst = dst + (32 - (dst & 31)) | |
| XVMOVQ X1, (R4) | |
| SUBV $32, R6 | |
| ADDV $32, R4 | |
| ADDV $32, R5 | |
| lasx_forward_move_256: | |
| SGTU $256, R6, R9 | |
| BNE R9, lasx_forward_move_64 | |
| lasx_forward_move_256loop: | |
| SUBV $256, R6 | |
| SGTU $256, R6, R9 | |
| XVMOVQ 0(R5), X0 | |
| XVMOVQ 32(R5), X1 | |
| XVMOVQ 64(R5), X2 | |
| XVMOVQ 96(R5), X3 | |
| XVMOVQ 128(R5), X4 | |
| XVMOVQ 160(R5), X5 | |
| XVMOVQ 192(R5), X6 | |
| XVMOVQ 224(R5), X7 | |
| XVMOVQ X0, 0(R4) | |
| XVMOVQ X1, 32(R4) | |
| XVMOVQ X2, 64(R4) | |
| XVMOVQ X3, 96(R4) | |
| XVMOVQ X4, 128(R4) | |
| XVMOVQ X5, 160(R4) | |
| XVMOVQ X6, 192(R4) | |
| XVMOVQ X7, 224(R4) | |
| ADDV $256, R5 | |
| ADDV $256, R4 | |
| BEQ R9, lasx_forward_move_256loop | |
| lasx_forward_move_64: | |
| SGTU $64, R6, R9 | |
| BNE R9, lasx_forward_move_tail | |
| lasx_forward_move_64loop: | |
| SUBV $64, R6 | |
| SGTU $64, R6, R9 | |
| XVMOVQ (R5), X0 | |
| XVMOVQ 32(R5), X1 | |
| XVMOVQ X0, (R4) | |
| XVMOVQ X1, 32(R4) | |
| ADDV $64, R5 | |
| ADDV $64, R4 | |
| BEQ R9, lasx_forward_move_64loop | |
| lasx_forward_move_tail: | |
| // 0 < remaining_length < 64 | |
| BNE R6, tail | |
| RET | |
| lasx_backward: | |
| AND $31, R7, R9 // dstend & 31 | |
| BEQ R9, lasx_backward_move_256 | |
| lasx_backward_unaligned: | |
| XVMOVQ -32(R8), X0 | |
| SUBV R9, R6 // newcount = count - (dstend & 31) | |
| SUBV R9, R8 // newsrcend = srcend - (dstend & 31) | |
| XVMOVQ -32(R8), X1 | |
| XVMOVQ X0, -32(R7) | |
| SUBV R9, R7 // newdstend = dstend - (dstend & 31) | |
| XVMOVQ X1, -32(R7) | |
| SUBV $32, R6 | |
| SUBV $32, R7 | |
| SUBV $32, R8 | |
| lasx_backward_move_256: | |
| SGTU $256, R6, R9 | |
| BNE R9, lasx_backward_move_64 | |
| lasx_backward_move_256loop: | |
| SUBV $256, R6 | |
| SGTU $256, R6, R9 | |
| XVMOVQ -32(R8), X0 | |
| XVMOVQ -64(R8), X1 | |
| XVMOVQ -96(R8), X2 | |
| XVMOVQ -128(R8), X3 | |
| XVMOVQ -160(R8), X4 | |
| XVMOVQ -192(R8), X5 | |
| XVMOVQ -224(R8), X6 | |
| XVMOVQ -256(R8), X7 | |
| XVMOVQ X0, -32(R7) | |
| XVMOVQ X1, -64(R7) | |
| XVMOVQ X2, -96(R7) | |
| XVMOVQ X3, -128(R7) | |
| XVMOVQ X4, -160(R7) | |
| XVMOVQ X5, -192(R7) | |
| XVMOVQ X6, -224(R7) | |
| XVMOVQ X7, -256(R7) | |
| SUBV $256, R8 | |
| SUBV $256, R7 | |
| BEQ R9, lasx_backward_move_256loop | |
| lasx_backward_move_64: | |
| SGTU $64, R6, R9 | |
| BNE R9, lasx_backward_move_tail | |
| lasx_backward_move_64loop: | |
| SUBV $64, R6 | |
| SGTU $64, R6, R9 | |
| XVMOVQ -32(R8), X0 | |
| XVMOVQ -64(R8), X1 | |
| XVMOVQ X0, -32(R7) | |
| XVMOVQ X1, -64(R7) | |
| SUBV $64, R8 | |
| SUBV $64, R7 | |
| BEQ R9, lasx_backward_move_64loop | |
| lasx_backward_move_tail: | |
| // 0 < remaining_length < 64 | |
| BNE R6, tail | |
| RET |