| // Copyright 2022 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| #include "go_asm.h" |
| #include "textflag.h" |
| |
| // Register map |
| // |
| // R4: ptr |
| // R5: n |
| // R6: ptrend |
| // R7: tmp |
| |
| // Algorithm: |
| // |
| // 1. if lasx is enabled: |
| // THRESHOLD = 256, ALIGNMENTS = 32, LOOPBLOCKS = 256, |
| // else if lsx is enabled: |
| // THRESHOLD = 128, ALIGNMENTS = 16, LOOPBLOCKS = 128, |
| // else |
| // THRESHOLD = 64, ALIGNMENTS = 8, LOOPBLOCKS = 64, |
| // |
| // 2. when 'count <= THRESHOLD' bytes, memory alignment check is omitted. |
| // The handling is divided into distinct cases based on the size of count: |
| // a. clr_0, clr_1, clr_2, clr_3, clr_4, clr_5through7, clr_8, |
| // clr_9through16, clr_17through32, clr_33through64, |
| // b. lsx_clr_17through32, lsx_clr_33through64, lsx_clr_65through128, |
| // c. lasx_clr_17through32, lasx_clr_33through64, lsx_clr_65through128, |
| // lasx_clr_65through128, lasx_clr_129through256 |
| // |
| // 3. when 'count > THRESHOLD' bytes, memory alignment check is performed. Unaligned |
| // bytes are processed first (that is, ALIGNMENTS - (ptr & (ALIGNMENTS-1))), and then |
| // a LOOPBLOCKS-byte loop is executed to zero out memory. |
| // When the number of remaining bytes not cleared is n < LOOPBLOCKS bytes, a tail |
| // processing is performed, invoking the corresponding case based on the size of n. |
| // |
| // example: |
| // THRESHOLD = 64, ALIGNMENTS = 8, LOOPBLOCKS = 64 |
| // |
| // ptr newptr ptrend |
| // | |<----count after correction---->| |
| // |<-------------count before correction---------->| |
| // |<--8-(ptr&7)-->| |<---64 bytes--->| |
| // +------------------------------------------------+ |
| // | Head | Body | Tail | |
| // +---------------+---------------+----------------+ |
| // newptr = ptr - (ptr & 7) + 8 |
| // count = count - 8 + (ptr & 7) |
| |
| // func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) |
| TEXT runtime·memclrNoHeapPointers<ABIInternal>(SB),NOSPLIT,$0-16 |
| BEQ R5, clr_0 |
| ADDV R4, R5, R6 |
| tail: |
| // <=64 bytes, clear directly, not check aligned |
| SGTU $2, R5, R7 |
| BNE R7, clr_1 |
| SGTU $3, R5, R7 |
| BNE R7, clr_2 |
| SGTU $4, R5, R7 |
| BNE R7, clr_3 |
| SGTU $5, R5, R7 |
| BNE R7, clr_4 |
| SGTU $8, R5, R7 |
| BNE R7, clr_5through7 |
| SGTU $9, R5, R7 |
| BNE R7, clr_8 |
| SGTU $17, R5, R7 |
| BNE R7, clr_9through16 |
| |
| MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLASX(SB), R7 |
| BNE R7, lasx_tail |
| MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R7 |
| BNE R7, lsx_tail |
| |
| SGTU $33, R5, R7 |
| BNE R7, clr_17through32 |
| SGTU $65, R5, R7 |
| BNE R7, clr_33through64 |
| JMP clr_large |
| |
| lasx_tail: |
| // X0 = 0 |
| XVXORV X0, X0, X0 |
| |
| SGTU $33, R5, R7 |
| BNE R7, lasx_clr_17through32 |
| SGTU $65, R5, R7 |
| BNE R7, lasx_clr_33through64 |
| SGTU $129, R5, R7 |
| BNE R7, lasx_clr_65through128 |
| SGTU $257, R5, R7 |
| BNE R7, lasx_clr_129through256 |
| JMP lasx_clr_large |
| |
| lsx_tail: |
| // V0 = 0 |
| VXORV V0, V0, V0 |
| |
| SGTU $33, R5, R7 |
| BNE R7, lsx_clr_17through32 |
| SGTU $65, R5, R7 |
| BNE R7, lsx_clr_33through64 |
| SGTU $129, R5, R7 |
| BNE R7, lsx_clr_65through128 |
| JMP lsx_clr_large |
| |
| // use simd 256 instructions to implement memclr |
| // n > 256 bytes, check 32-byte alignment |
| lasx_clr_large: |
| AND $31, R4, R7 |
| BEQ R7, lasx_clr_256loop |
| XVMOVQ X0, (R4) |
| SUBV R7, R4 |
| ADDV R7, R5 |
| SUBV $32, R5 // newn = n - (32 - (ptr & 31)) |
| ADDV $32, R4 // newptr = ptr + (32 - (ptr & 31)) |
| SGTU $257, R5, R7 |
| BNE R7, lasx_clr_129through256 |
| lasx_clr_256loop: |
| SUBV $256, R5 |
| SGTU $256, R5, R7 |
| XVMOVQ X0, 0(R4) |
| XVMOVQ X0, 32(R4) |
| XVMOVQ X0, 64(R4) |
| XVMOVQ X0, 96(R4) |
| XVMOVQ X0, 128(R4) |
| XVMOVQ X0, 160(R4) |
| XVMOVQ X0, 192(R4) |
| XVMOVQ X0, 224(R4) |
| ADDV $256, R4 |
| BEQ R7, lasx_clr_256loop |
| |
| // remaining_length is 0 |
| BEQ R5, clr_0 |
| |
| // 128 < remaining_length < 256 |
| SGTU $129, R5, R7 |
| BEQ R7, lasx_clr_129through256 |
| |
| // 64 < remaining_length <= 128 |
| SGTU $65, R5, R7 |
| BEQ R7, lasx_clr_65through128 |
| |
| // 32 < remaining_length <= 64 |
| SGTU $33, R5, R7 |
| BEQ R7, lasx_clr_33through64 |
| |
| // 16 < remaining_length <= 32 |
| SGTU $17, R5, R7 |
| BEQ R7, lasx_clr_17through32 |
| |
| // 0 < remaining_length <= 16 |
| JMP tail |
| |
| // use simd 128 instructions to implement memclr |
| // n > 128 bytes, check 16-byte alignment |
| lsx_clr_large: |
| // check 16-byte alignment |
| AND $15, R4, R7 |
| BEQ R7, lsx_clr_128loop |
| VMOVQ V0, (R4) |
| SUBV R7, R4 |
| ADDV R7, R5 |
| SUBV $16, R5 // newn = n - (16 - (ptr & 15)) |
| ADDV $16, R4 // newptr = ptr + (16 - (ptr & 15)) |
| SGTU $129, R5, R7 |
| BNE R7, lsx_clr_65through128 |
| lsx_clr_128loop: |
| SUBV $128, R5 |
| SGTU $128, R5, R7 |
| VMOVQ V0, 0(R4) |
| VMOVQ V0, 16(R4) |
| VMOVQ V0, 32(R4) |
| VMOVQ V0, 48(R4) |
| VMOVQ V0, 64(R4) |
| VMOVQ V0, 80(R4) |
| VMOVQ V0, 96(R4) |
| VMOVQ V0, 112(R4) |
| ADDV $128, R4 |
| BEQ R7, lsx_clr_128loop |
| |
| // remaining_length is 0 |
| BEQ R5, clr_0 |
| |
| // 64 < remaining_length <= 128 |
| SGTU $65, R5, R7 |
| BEQ R7, lsx_clr_65through128 |
| |
| // 32 < remaining_length <= 64 |
| SGTU $33, R5, R7 |
| BEQ R7, lsx_clr_33through64 |
| |
| // 16 < remaining_length <= 32 |
| SGTU $17, R5, R7 |
| BEQ R7, lsx_clr_17through32 |
| |
| // 0 < remaining_length <= 16 |
| JMP tail |
| |
| // use general instructions to implement memclr |
| // n > 64 bytes, check 16-byte alignment |
| clr_large: |
| AND $7, R4, R7 |
| BEQ R7, clr_64loop |
| MOVV R0, (R4) |
| SUBV R7, R4 |
| ADDV R7, R5 |
| ADDV $8, R4 // newptr = ptr + (8 - (ptr & 7)) |
| SUBV $8, R5 // newn = n - (8 - (ptr & 7)) |
| MOVV $64, R7 |
| BLT R5, R7, clr_33through64 |
| clr_64loop: |
| SUBV $64, R5 |
| SGTU $64, R5, R7 |
| MOVV R0, (R4) |
| MOVV R0, 8(R4) |
| MOVV R0, 16(R4) |
| MOVV R0, 24(R4) |
| MOVV R0, 32(R4) |
| MOVV R0, 40(R4) |
| MOVV R0, 48(R4) |
| MOVV R0, 56(R4) |
| ADDV $64, R4 |
| BEQ R7, clr_64loop |
| |
| // remaining_length is 0 |
| BEQ R5, clr_0 |
| |
| // 32 < remaining_length < 64 |
| SGTU $33, R5, R7 |
| BEQ R7, clr_33through64 |
| |
| // 16 < remaining_length <= 32 |
| SGTU $17, R5, R7 |
| BEQ R7, clr_17through32 |
| |
| // 0 < remaining_length <= 16 |
| JMP tail |
| |
| clr_0: |
| RET |
| clr_1: |
| MOVB R0, (R4) |
| RET |
| clr_2: |
| MOVH R0, (R4) |
| RET |
| clr_3: |
| MOVH R0, (R4) |
| MOVB R0, 2(R4) |
| RET |
| clr_4: |
| MOVW R0, (R4) |
| RET |
| clr_5through7: |
| MOVW R0, (R4) |
| MOVW R0, -4(R6) |
| RET |
| clr_8: |
| MOVV R0, (R4) |
| RET |
| clr_9through16: |
| MOVV R0, (R4) |
| MOVV R0, -8(R6) |
| RET |
| clr_17through32: |
| MOVV R0, (R4) |
| MOVV R0, 8(R4) |
| MOVV R0, -16(R6) |
| MOVV R0, -8(R6) |
| RET |
| clr_33through64: |
| MOVV R0, (R4) |
| MOVV R0, 8(R4) |
| MOVV R0, 16(R4) |
| MOVV R0, 24(R4) |
| MOVV R0, -32(R6) |
| MOVV R0, -24(R6) |
| MOVV R0, -16(R6) |
| MOVV R0, -8(R6) |
| RET |
| |
| lasx_clr_17through32: |
| VMOVQ V0, 0(R4) |
| VMOVQ V0, -16(R6) |
| RET |
| lasx_clr_33through64: |
| XVMOVQ X0, 0(R4) |
| XVMOVQ X0, -32(R6) |
| RET |
| lasx_clr_65through128: |
| XVMOVQ X0, 0(R4) |
| XVMOVQ X0, 32(R4) |
| XVMOVQ X0, -64(R6) |
| XVMOVQ X0, -32(R6) |
| RET |
| lasx_clr_129through256: |
| XVMOVQ X0, 0(R4) |
| XVMOVQ X0, 32(R4) |
| XVMOVQ X0, 64(R4) |
| XVMOVQ X0, 96(R4) |
| XVMOVQ X0, -128(R6) |
| XVMOVQ X0, -96(R6) |
| XVMOVQ X0, -64(R6) |
| XVMOVQ X0, -32(R6) |
| RET |
| |
| lsx_clr_17through32: |
| VMOVQ V0, 0(R4) |
| VMOVQ V0, -16(R6) |
| RET |
| lsx_clr_33through64: |
| VMOVQ V0, 0(R4) |
| VMOVQ V0, 16(R4) |
| VMOVQ V0, -32(R6) |
| VMOVQ V0, -16(R6) |
| RET |
| lsx_clr_65through128: |
| VMOVQ V0, 0(R4) |
| VMOVQ V0, 16(R4) |
| VMOVQ V0, 32(R4) |
| VMOVQ V0, 48(R4) |
| VMOVQ V0, -64(R6) |
| VMOVQ V0, -48(R6) |
| VMOVQ V0, -32(R6) |
| VMOVQ V0, -16(R6) |
| RET |
