| // Copyright 2014 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| #include "textflag.h" |
| |
| // See memmove Go doc for important implementation constraints. |
| |
| // Register map |
| // |
| // dstin R0 |
| // src R1 |
| // count R2 |
| // dst R3 (same as R0, but gets modified in unaligned cases) |
| // srcend R4 |
| // dstend R5 |
| // data R6-R17 |
| // tmp1 R14 |
| |
| // Copies are split into 3 main cases: small copies of up to 32 bytes, medium |
| // copies of up to 128 bytes, and large copies. The overhead of the overlap |
| // check is negligible since it is only required for large copies. |
| // |
| // Large copies use a software pipelined loop processing 64 bytes per iteration. |
| // The destination pointer is 16-byte aligned to minimize unaligned accesses. |
| // The loop tail is handled by always copying 64 bytes from the end. |
| |
| // func memmove(to, from unsafe.Pointer, n uintptr) |
| TEXT runtime·memmove(SB), NOSPLIT|NOFRAME, $0-24 |
| MOVD to+0(FP), R0 |
| MOVD from+8(FP), R1 |
| MOVD n+16(FP), R2 |
| CBZ R2, copy0 |
| |
| // Small copies: 1..16 bytes |
| CMP $16, R2 |
| BLE copy16 |
| |
| // Large copies |
| CMP $128, R2 |
| BHI copy_long |
| CMP $32, R2 |
| BHI copy32_128 |
| |
| // Small copies: 17..32 bytes. |
| LDP (R1), (R6, R7) |
| ADD R1, R2, R4 // R4 points just past the last source byte |
| LDP -16(R4), (R12, R13) |
| STP (R6, R7), (R0) |
| ADD R0, R2, R5 // R5 points just past the last destination byte |
| STP (R12, R13), -16(R5) |
| RET |
| |
| // Small copies: 1..16 bytes. |
| copy16: |
| ADD R1, R2, R4 // R4 points just past the last source byte |
| ADD R0, R2, R5 // R5 points just past the last destination byte |
| CMP $8, R2 |
| BLT copy7 |
| MOVD (R1), R6 |
| MOVD -8(R4), R7 |
| MOVD R6, (R0) |
| MOVD R7, -8(R5) |
| RET |
| |
| copy7: |
| TBZ $2, R2, copy3 |
| MOVWU (R1), R6 |
| MOVWU -4(R4), R7 |
| MOVW R6, (R0) |
| MOVW R7, -4(R5) |
| RET |
| |
| copy3: |
| TBZ $1, R2, copy1 |
| MOVHU (R1), R6 |
| MOVHU -2(R4), R7 |
| MOVH R6, (R0) |
| MOVH R7, -2(R5) |
| RET |
| |
| copy1: |
| MOVBU (R1), R6 |
| MOVB R6, (R0) |
| |
| copy0: |
| RET |
| |
| // Medium copies: 33..128 bytes. |
| copy32_128: |
| ADD R1, R2, R4 // R4 points just past the last source byte |
| ADD R0, R2, R5 // R5 points just past the last destination byte |
| LDP (R1), (R6, R7) |
| LDP 16(R1), (R8, R9) |
| LDP -32(R4), (R10, R11) |
| LDP -16(R4), (R12, R13) |
| CMP $64, R2 |
| BHI copy128 |
| STP (R6, R7), (R0) |
| STP (R8, R9), 16(R0) |
| STP (R10, R11), -32(R5) |
| STP (R12, R13), -16(R5) |
| RET |
| |
| // Copy 65..128 bytes. |
| copy128: |
| LDP 32(R1), (R14, R15) |
| LDP 48(R1), (R16, R17) |
| CMP $96, R2 |
| BLS copy96 |
| LDP -64(R4), (R2, R3) |
| LDP -48(R4), (R1, R4) |
| STP (R2, R3), -64(R5) |
| STP (R1, R4), -48(R5) |
| |
| copy96: |
| STP (R6, R7), (R0) |
| STP (R8, R9), 16(R0) |
| STP (R14, R15), 32(R0) |
| STP (R16, R17), 48(R0) |
| STP (R10, R11), -32(R5) |
| STP (R12, R13), -16(R5) |
| RET |
| |
| // Copy more than 128 bytes. |
| copy_long: |
| ADD R1, R2, R4 // R4 points just past the last source byte |
| ADD R0, R2, R5 // R5 points just past the last destination byte |
| MOVD ZR, R7 |
| MOVD ZR, R8 |
| |
| CMP $1024, R2 |
| BLT backward_check |
| // feature detect to decide how to align |
| MOVBU runtime·arm64UseAlignedLoads(SB), R6 |
| CBNZ R6, use_aligned_loads |
| MOVD R0, R7 |
| MOVD R5, R8 |
| B backward_check |
| use_aligned_loads: |
| MOVD R1, R7 |
| MOVD R4, R8 |
| // R7 and R8 are used here for the realignment calculation. In |
| // the use_aligned_loads case, R7 is the src pointer and R8 is |
| // srcend pointer, which is used in the backward copy case. |
| // When doing aligned stores, R7 is the dst pointer and R8 is |
| // the dstend pointer. |
| |
| backward_check: |
| // Use backward copy if there is an overlap. |
| SUB R1, R0, R14 |
| CBZ R14, copy0 |
| CMP R2, R14 |
| BCC copy_long_backward |
| |
| // Copy 16 bytes and then align src (R1) or dst (R0) to 16-byte alignment. |
| LDP (R1), (R12, R13) // Load A |
| AND $15, R7, R14 // Calculate the realignment offset |
| SUB R14, R1, R1 |
| SUB R14, R0, R3 // move dst back same amount as src |
| ADD R14, R2, R2 |
| LDP 16(R1), (R6, R7) // Load B |
| STP (R12, R13), (R0) // Store A |
| LDP 32(R1), (R8, R9) // Load C |
| LDP 48(R1), (R10, R11) // Load D |
| LDP.W 64(R1), (R12, R13) // Load E |
| // 80 bytes have been loaded; if less than 80+64 bytes remain, copy from the end |
| SUBS $144, R2, R2 |
| BLS copy64_from_end |
| |
| loop64: |
| STP (R6, R7), 16(R3) // Store B |
| LDP 16(R1), (R6, R7) // Load B (next iteration) |
| STP (R8, R9), 32(R3) // Store C |
| LDP 32(R1), (R8, R9) // Load C |
| STP (R10, R11), 48(R3) // Store D |
| LDP 48(R1), (R10, R11) // Load D |
| STP.W (R12, R13), 64(R3) // Store E |
| LDP.W 64(R1), (R12, R13) // Load E |
| SUBS $64, R2, R2 |
| BHI loop64 |
| |
| // Write the last iteration and copy 64 bytes from the end. |
| copy64_from_end: |
| LDP -64(R4), (R14, R15) // Load F |
| STP (R6, R7), 16(R3) // Store B |
| LDP -48(R4), (R6, R7) // Load G |
| STP (R8, R9), 32(R3) // Store C |
| LDP -32(R4), (R8, R9) // Load H |
| STP (R10, R11), 48(R3) // Store D |
| LDP -16(R4), (R10, R11) // Load I |
| STP (R12, R13), 64(R3) // Store E |
| STP (R14, R15), -64(R5) // Store F |
| STP (R6, R7), -48(R5) // Store G |
| STP (R8, R9), -32(R5) // Store H |
| STP (R10, R11), -16(R5) // Store I |
| RET |
| |
| // Large backward copy for overlapping copies. |
| // Copy 16 bytes and then align srcend (R4) or dstend (R5) to 16-byte alignment. |
| copy_long_backward: |
| LDP -16(R4), (R12, R13) |
| AND $15, R8, R14 |
| SUB R14, R4, R4 |
| SUB R14, R2, R2 |
| LDP -16(R4), (R6, R7) |
| STP (R12, R13), -16(R5) |
| LDP -32(R4), (R8, R9) |
| LDP -48(R4), (R10, R11) |
| LDP.W -64(R4), (R12, R13) |
| SUB R14, R5, R5 |
| SUBS $128, R2, R2 |
| BLS copy64_from_start |
| |
| loop64_backward: |
| STP (R6, R7), -16(R5) |
| LDP -16(R4), (R6, R7) |
| STP (R8, R9), -32(R5) |
| LDP -32(R4), (R8, R9) |
| STP (R10, R11), -48(R5) |
| LDP -48(R4), (R10, R11) |
| STP.W (R12, R13), -64(R5) |
| LDP.W -64(R4), (R12, R13) |
| SUBS $64, R2, R2 |
| BHI loop64_backward |
| |
| // Write the last iteration and copy 64 bytes from the start. |
| copy64_from_start: |
| LDP 48(R1), (R2, R3) |
| STP (R6, R7), -16(R5) |
| LDP 32(R1), (R6, R7) |
| STP (R8, R9), -32(R5) |
| LDP 16(R1), (R8, R9) |
| STP (R10, R11), -48(R5) |
| LDP (R1), (R10, R11) |
| STP (R12, R13), -64(R5) |
| STP (R2, R3), 48(R0) |
| STP (R6, R7), 32(R0) |
| STP (R8, R9), 16(R0) |
| STP (R10, R11), (R0) |
| RET |
