| // Copyright 2014 The Go Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style | |
| // license that can be found in the LICENSE file. | |
| #include "textflag.h" | |
| // func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) | |
| TEXT runtime·memclrNoHeapPointers(SB),NOSPLIT,$0-16 | |
| MOVD ptr+0(FP), R0 | |
| MOVD n+8(FP), R1 | |
| CMP $16, R1 | |
| // If n is equal to 16 bytes, use zero_exact_16 to zero | |
| BEQ zero_exact_16 | |
| // If n is greater than 16 bytes, use zero_by_16 to zero | |
| BHI zero_by_16 | |
| // n is less than 16 bytes | |
| ADD R1, R0, R7 | |
| TBZ $3, R1, less_than_8 | |
| MOVD ZR, (R0) | |
| MOVD ZR, -8(R7) | |
| RET | |
| less_than_8: | |
| TBZ $2, R1, less_than_4 | |
| MOVW ZR, (R0) | |
| MOVW ZR, -4(R7) | |
| RET | |
| less_than_4: | |
| CBZ R1, ending | |
| MOVB ZR, (R0) | |
| TBZ $1, R1, ending | |
| MOVH ZR, -2(R7) | |
| ending: | |
| RET | |
| zero_exact_16: | |
| // n is exactly 16 bytes | |
| STP (ZR, ZR), (R0) | |
| RET | |
| zero_by_16: | |
| // n greater than 16 bytes, check if the start address is aligned | |
| NEG R0, R4 | |
| ANDS $15, R4, R4 | |
| // Try zeroing using zva if the start address is aligned with 16 | |
| BEQ try_zva | |
| // Non-aligned store | |
| STP (ZR, ZR), (R0) | |
| // Make the destination aligned | |
| SUB R4, R1, R1 | |
| ADD R4, R0, R0 | |
| B try_zva | |
| tail_maybe_long: | |
| CMP $64, R1 | |
| BHS no_zva | |
| tail63: | |
| ANDS $48, R1, R3 | |
| BEQ last16 | |
| CMPW $32, R3 | |
| BEQ last48 | |
| BLT last32 | |
| STP.P (ZR, ZR), 16(R0) | |
| last48: | |
| STP.P (ZR, ZR), 16(R0) | |
| last32: | |
| STP.P (ZR, ZR), 16(R0) | |
| // The last store length is at most 16, so it is safe to use | |
| // stp to write last 16 bytes | |
| last16: | |
| ANDS $15, R1, R1 | |
| CBZ R1, last_end | |
| ADD R1, R0, R0 | |
| STP (ZR, ZR), -16(R0) | |
| last_end: | |
| RET | |
| no_zva: | |
| SUB $16, R0, R0 | |
| SUB $64, R1, R1 | |
| loop_64: | |
| STP (ZR, ZR), 16(R0) | |
| STP (ZR, ZR), 32(R0) | |
| STP (ZR, ZR), 48(R0) | |
| STP.W (ZR, ZR), 64(R0) | |
| SUBS $64, R1, R1 | |
| BGE loop_64 | |
| ANDS $63, R1, ZR | |
| ADD $16, R0, R0 | |
| BNE tail63 | |
| RET | |
| try_zva: | |
| // Try using the ZVA feature to zero entire cache lines | |
| // It is not meaningful to use ZVA if the block size is less than 64, | |
| // so make sure that n is greater than or equal to 64 | |
| CMP $63, R1 | |
| BLE tail63 | |
| CMP $128, R1 | |
| // Ensure n is at least 128 bytes, so that there is enough to copy after | |
| // alignment. | |
| BLT no_zva | |
| // Check if ZVA is allowed from user code, and if so get the block size | |
| MOVW block_size<>(SB), R5 | |
| TBNZ $31, R5, no_zva | |
| CBNZ R5, zero_by_line | |
| // DCZID_EL0 bit assignments | |
| // [63:5] Reserved | |
| // [4] DZP, if bit set DC ZVA instruction is prohibited, else permitted | |
| // [3:0] log2 of the block size in words, eg. if it returns 0x4 then block size is 16 words | |
| MRS DCZID_EL0, R3 | |
| TBZ $4, R3, init | |
| // ZVA not available | |
| MOVW $~0, R5 | |
| MOVW R5, block_size<>(SB) | |
| B no_zva | |
| init: | |
| MOVW $4, R9 | |
| ANDW $15, R3, R5 | |
| LSLW R5, R9, R5 | |
| MOVW R5, block_size<>(SB) | |
| ANDS $63, R5, R9 | |
| // Block size is less than 64. | |
| BNE no_zva | |
| zero_by_line: | |
| CMP R5, R1 | |
| // Not enough memory to reach alignment | |
| BLO no_zva | |
| SUB $1, R5, R6 | |
| NEG R0, R4 | |
| ANDS R6, R4, R4 | |
| // Already aligned | |
| BEQ aligned | |
| // check there is enough to copy after alignment | |
| SUB R4, R1, R3 | |
| // Check that the remaining length to ZVA after alignment | |
| // is greater than 64. | |
| CMP $64, R3 | |
| CCMP GE, R3, R5, $10 // condition code GE, NZCV=0b1010 | |
| BLT no_zva | |
| // We now have at least 64 bytes to zero, update n | |
| MOVD R3, R1 | |
| loop_zva_prolog: | |
| STP (ZR, ZR), (R0) | |
| STP (ZR, ZR), 16(R0) | |
| STP (ZR, ZR), 32(R0) | |
| SUBS $64, R4, R4 | |
| STP (ZR, ZR), 48(R0) | |
| ADD $64, R0, R0 | |
| BGE loop_zva_prolog | |
| ADD R4, R0, R0 | |
| aligned: | |
| SUB R5, R1, R1 | |
| loop_zva: | |
| WORD $0xd50b7420 // DC ZVA, R0 | |
| ADD R5, R0, R0 | |
| SUBS R5, R1, R1 | |
| BHS loop_zva | |
| ANDS R6, R1, R1 | |
| BNE tail_maybe_long | |
| RET | |
| GLOBL block_size<>(SB), NOPTR, $8 |