| // Copyright 2022 The Go Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style | |
| // license that can be found in the LICENSE file. | |
| #include "go_asm.h" | |
| #include "textflag.h" | |
| // Register map | |
| // | |
| // R4: ptr | |
| // R5: n | |
| // R6: ptrend | |
| // R7: tmp | |
| // Algorithm: | |
| // | |
| // 1. if lasx is enabled: | |
| // THRESHOLD = 256, ALIGNMENTS = 32, LOOPBLOCKS = 256, | |
| // else if lsx is enabled: | |
| // THRESHOLD = 128, ALIGNMENTS = 16, LOOPBLOCKS = 128, | |
| // else | |
| // THRESHOLD = 64, ALIGNMENTS = 8, LOOPBLOCKS = 64, | |
| // | |
| // 2. when 'count <= THRESHOLD' bytes, memory alignment check is omitted. | |
| // The handling is divided into distinct cases based on the size of count: | |
| // a. clr_0, clr_1, clr_2, clr_3, clr_4, clr_5through7, clr_8, | |
| // clr_9through16, clr_17through32, clr_33through64, | |
| // b. lsx_clr_17through32, lsx_clr_33through64, lsx_clr_65through128, | |
| // c. lasx_clr_17through32, lasx_clr_33through64, lsx_clr_65through128, | |
| // lasx_clr_65through128, lasx_clr_129through256 | |
| // | |
| // 3. when 'count > THRESHOLD' bytes, memory alignment check is performed. Unaligned | |
| // bytes are processed first (that is, ALIGNMENTS - (ptr & (ALIGNMENTS-1))), and then | |
| // a LOOPBLOCKS-byte loop is executed to zero out memory. | |
| // When the number of remaining bytes not cleared is n < LOOPBLOCKS bytes, a tail | |
| // processing is performed, invoking the corresponding case based on the size of n. | |
| // | |
| // example: | |
| // THRESHOLD = 64, ALIGNMENTS = 8, LOOPBLOCKS = 64 | |
| // | |
| // ptr newptr ptrend | |
| // | |<----count after correction---->| | |
| // |<-------------count before correction---------->| | |
| // |<--8-(ptr&7)-->| |<---64 bytes--->| | |
| // +------------------------------------------------+ | |
| // | Head | Body | Tail | | |
| // +---------------+---------------+----------------+ | |
| // newptr = ptr - (ptr & 7) + 8 | |
| // count = count - 8 + (ptr & 7) | |
| // func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) | |
| TEXT runtime·memclrNoHeapPointers<ABIInternal>(SB),NOSPLIT,$0-16 | |
| BEQ R5, clr_0 | |
| ADDV R4, R5, R6 | |
| tail: | |
| // <=64 bytes, clear directly, not check aligned | |
| SGTU $2, R5, R7 | |
| BNE R7, clr_1 | |
| SGTU $3, R5, R7 | |
| BNE R7, clr_2 | |
| SGTU $4, R5, R7 | |
| BNE R7, clr_3 | |
| SGTU $5, R5, R7 | |
| BNE R7, clr_4 | |
| SGTU $8, R5, R7 | |
| BNE R7, clr_5through7 | |
| SGTU $9, R5, R7 | |
| BNE R7, clr_8 | |
| SGTU $17, R5, R7 | |
| BNE R7, clr_9through16 | |
| MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLASX(SB), R7 | |
| BNE R7, lasx_tail | |
| MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R7 | |
| BNE R7, lsx_tail | |
| SGTU $33, R5, R7 | |
| BNE R7, clr_17through32 | |
| SGTU $65, R5, R7 | |
| BNE R7, clr_33through64 | |
| JMP clr_large | |
| lasx_tail: | |
| // X0 = 0 | |
| XVXORV X0, X0, X0 | |
| SGTU $33, R5, R7 | |
| BNE R7, lasx_clr_17through32 | |
| SGTU $65, R5, R7 | |
| BNE R7, lasx_clr_33through64 | |
| SGTU $129, R5, R7 | |
| BNE R7, lasx_clr_65through128 | |
| SGTU $257, R5, R7 | |
| BNE R7, lasx_clr_129through256 | |
| JMP lasx_clr_large | |
| lsx_tail: | |
| // V0 = 0 | |
| VXORV V0, V0, V0 | |
| SGTU $33, R5, R7 | |
| BNE R7, lsx_clr_17through32 | |
| SGTU $65, R5, R7 | |
| BNE R7, lsx_clr_33through64 | |
| SGTU $129, R5, R7 | |
| BNE R7, lsx_clr_65through128 | |
| JMP lsx_clr_large | |
| // use simd 256 instructions to implement memclr | |
| // n > 256 bytes, check 32-byte alignment | |
| lasx_clr_large: | |
| AND $31, R4, R7 | |
| BEQ R7, lasx_clr_256loop | |
| XVMOVQ X0, (R4) | |
| SUBV R7, R4 | |
| ADDV R7, R5 | |
| SUBV $32, R5 // newn = n - (32 - (ptr & 31)) | |
| ADDV $32, R4 // newptr = ptr + (32 - (ptr & 31)) | |
| SGTU $257, R5, R7 | |
| BNE R7, lasx_clr_129through256 | |
| lasx_clr_256loop: | |
| SUBV $256, R5 | |
| SGTU $256, R5, R7 | |
| XVMOVQ X0, 0(R4) | |
| XVMOVQ X0, 32(R4) | |
| XVMOVQ X0, 64(R4) | |
| XVMOVQ X0, 96(R4) | |
| XVMOVQ X0, 128(R4) | |
| XVMOVQ X0, 160(R4) | |
| XVMOVQ X0, 192(R4) | |
| XVMOVQ X0, 224(R4) | |
| ADDV $256, R4 | |
| BEQ R7, lasx_clr_256loop | |
| // remaining_length is 0 | |
| BEQ R5, clr_0 | |
| // 128 < remaining_length < 256 | |
| SGTU $129, R5, R7 | |
| BEQ R7, lasx_clr_129through256 | |
| // 64 < remaining_length <= 128 | |
| SGTU $65, R5, R7 | |
| BEQ R7, lasx_clr_65through128 | |
| // 32 < remaining_length <= 64 | |
| SGTU $33, R5, R7 | |
| BEQ R7, lasx_clr_33through64 | |
| // 16 < remaining_length <= 32 | |
| SGTU $17, R5, R7 | |
| BEQ R7, lasx_clr_17through32 | |
| // 0 < remaining_length <= 16 | |
| JMP tail | |
| // use simd 128 instructions to implement memclr | |
| // n > 128 bytes, check 16-byte alignment | |
| lsx_clr_large: | |
| // check 16-byte alignment | |
| AND $15, R4, R7 | |
| BEQ R7, lsx_clr_128loop | |
| VMOVQ V0, (R4) | |
| SUBV R7, R4 | |
| ADDV R7, R5 | |
| SUBV $16, R5 // newn = n - (16 - (ptr & 15)) | |
| ADDV $16, R4 // newptr = ptr + (16 - (ptr & 15)) | |
| SGTU $129, R5, R7 | |
| BNE R7, lsx_clr_65through128 | |
| lsx_clr_128loop: | |
| SUBV $128, R5 | |
| SGTU $128, R5, R7 | |
| VMOVQ V0, 0(R4) | |
| VMOVQ V0, 16(R4) | |
| VMOVQ V0, 32(R4) | |
| VMOVQ V0, 48(R4) | |
| VMOVQ V0, 64(R4) | |
| VMOVQ V0, 80(R4) | |
| VMOVQ V0, 96(R4) | |
| VMOVQ V0, 112(R4) | |
| ADDV $128, R4 | |
| BEQ R7, lsx_clr_128loop | |
| // remaining_length is 0 | |
| BEQ R5, clr_0 | |
| // 64 < remaining_length <= 128 | |
| SGTU $65, R5, R7 | |
| BEQ R7, lsx_clr_65through128 | |
| // 32 < remaining_length <= 64 | |
| SGTU $33, R5, R7 | |
| BEQ R7, lsx_clr_33through64 | |
| // 16 < remaining_length <= 32 | |
| SGTU $17, R5, R7 | |
| BEQ R7, lsx_clr_17through32 | |
| // 0 < remaining_length <= 16 | |
| JMP tail | |
| // use general instructions to implement memclr | |
| // n > 64 bytes, check 16-byte alignment | |
| clr_large: | |
| AND $7, R4, R7 | |
| BEQ R7, clr_64loop | |
| MOVV R0, (R4) | |
| SUBV R7, R4 | |
| ADDV R7, R5 | |
| ADDV $8, R4 // newptr = ptr + (8 - (ptr & 7)) | |
| SUBV $8, R5 // newn = n - (8 - (ptr & 7)) | |
| MOVV $64, R7 | |
| BLT R5, R7, clr_33through64 | |
| clr_64loop: | |
| SUBV $64, R5 | |
| SGTU $64, R5, R7 | |
| MOVV R0, (R4) | |
| MOVV R0, 8(R4) | |
| MOVV R0, 16(R4) | |
| MOVV R0, 24(R4) | |
| MOVV R0, 32(R4) | |
| MOVV R0, 40(R4) | |
| MOVV R0, 48(R4) | |
| MOVV R0, 56(R4) | |
| ADDV $64, R4 | |
| BEQ R7, clr_64loop | |
| // remaining_length is 0 | |
| BEQ R5, clr_0 | |
| // 32 < remaining_length < 64 | |
| SGTU $33, R5, R7 | |
| BEQ R7, clr_33through64 | |
| // 16 < remaining_length <= 32 | |
| SGTU $17, R5, R7 | |
| BEQ R7, clr_17through32 | |
| // 0 < remaining_length <= 16 | |
| JMP tail | |
| clr_0: | |
| RET | |
| clr_1: | |
| MOVB R0, (R4) | |
| RET | |
| clr_2: | |
| MOVH R0, (R4) | |
| RET | |
| clr_3: | |
| MOVH R0, (R4) | |
| MOVB R0, 2(R4) | |
| RET | |
| clr_4: | |
| MOVW R0, (R4) | |
| RET | |
| clr_5through7: | |
| MOVW R0, (R4) | |
| MOVW R0, -4(R6) | |
| RET | |
| clr_8: | |
| MOVV R0, (R4) | |
| RET | |
| clr_9through16: | |
| MOVV R0, (R4) | |
| MOVV R0, -8(R6) | |
| RET | |
| clr_17through32: | |
| MOVV R0, (R4) | |
| MOVV R0, 8(R4) | |
| MOVV R0, -16(R6) | |
| MOVV R0, -8(R6) | |
| RET | |
| clr_33through64: | |
| MOVV R0, (R4) | |
| MOVV R0, 8(R4) | |
| MOVV R0, 16(R4) | |
| MOVV R0, 24(R4) | |
| MOVV R0, -32(R6) | |
| MOVV R0, -24(R6) | |
| MOVV R0, -16(R6) | |
| MOVV R0, -8(R6) | |
| RET | |
| lasx_clr_17through32: | |
| VMOVQ V0, 0(R4) | |
| VMOVQ V0, -16(R6) | |
| RET | |
| lasx_clr_33through64: | |
| XVMOVQ X0, 0(R4) | |
| XVMOVQ X0, -32(R6) | |
| RET | |
| lasx_clr_65through128: | |
| XVMOVQ X0, 0(R4) | |
| XVMOVQ X0, 32(R4) | |
| XVMOVQ X0, -64(R6) | |
| XVMOVQ X0, -32(R6) | |
| RET | |
| lasx_clr_129through256: | |
| XVMOVQ X0, 0(R4) | |
| XVMOVQ X0, 32(R4) | |
| XVMOVQ X0, 64(R4) | |
| XVMOVQ X0, 96(R4) | |
| XVMOVQ X0, -128(R6) | |
| XVMOVQ X0, -96(R6) | |
| XVMOVQ X0, -64(R6) | |
| XVMOVQ X0, -32(R6) | |
| RET | |
| lsx_clr_17through32: | |
| VMOVQ V0, 0(R4) | |
| VMOVQ V0, -16(R6) | |
| RET | |
| lsx_clr_33through64: | |
| VMOVQ V0, 0(R4) | |
| VMOVQ V0, 16(R4) | |
| VMOVQ V0, -32(R6) | |
| VMOVQ V0, -16(R6) | |
| RET | |
| lsx_clr_65through128: | |
| VMOVQ V0, 0(R4) | |
| VMOVQ V0, 16(R4) | |
| VMOVQ V0, 32(R4) | |
| VMOVQ V0, 48(R4) | |
| VMOVQ V0, -64(R6) | |
| VMOVQ V0, -48(R6) | |
| VMOVQ V0, -32(R6) | |
| VMOVQ V0, -16(R6) | |
| RET |