| // Copyright 2018 The Go Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style | |
| // license that can be found in the LICENSE file. | |
| //go:build ppc64 || ppc64le | |
| #include "go_asm.h" | |
| #include "textflag.h" | |
| TEXT ·IndexByte<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-40 | |
| // R3 = byte array pointer | |
| // R4 = length | |
| MOVD R6, R5 // R5 = byte | |
| BR indexbytebody<>(SB) | |
| TEXT ·IndexByteString<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32 | |
| // R3 = string | |
| // R4 = length | |
| // R5 = byte | |
| BR indexbytebody<>(SB) | |
| #ifndef GOPPC64_power9 | |
| #ifdef GOARCH_ppc64le | |
| DATA indexbytevbperm<>+0(SB)/8, $0x3830282018100800 | |
| DATA indexbytevbperm<>+8(SB)/8, $0x7870686058504840 | |
| #else | |
| DATA indexbytevbperm<>+0(SB)/8, $0x0008101820283038 | |
| DATA indexbytevbperm<>+8(SB)/8, $0x4048505860687078 | |
| #endif | |
| GLOBL indexbytevbperm<>+0(SB), RODATA, $16 | |
| #endif | |
| // Some operations are endian specific, choose the correct opcode base on GOARCH. | |
| // Note, _VCZBEBB is only available on power9 and newer. | |
| #ifdef GOARCH_ppc64le | |
| #define _LDBEX MOVDBR | |
| #define _LWBEX MOVWBR | |
| #define _LHBEX MOVHBR | |
| #define _VCZBEBB VCTZLSBB | |
| #else | |
| #define _LDBEX MOVD | |
| #define _LWBEX MOVW | |
| #define _LHBEX MOVH | |
| #define _VCZBEBB VCLZLSBB | |
| #endif | |
| // R3 = addr of string | |
| // R4 = len of string | |
| // R5 = byte to find | |
| // On exit: | |
| // R3 = return value | |
| TEXT indexbytebody<>(SB),NOSPLIT|NOFRAME,$0-0 | |
| CMPU R4,$32 | |
| #ifndef GOPPC64_power9 | |
| // Load VBPERMQ constant to reduce compare into an ordered bit mask. | |
| MOVD $indexbytevbperm<>+00(SB),R16 | |
| LXVD2X (R16),V0 // Set up swap string | |
| #endif | |
| MTVRD R5,V1 | |
| VSPLTB $7,V1,V1 // Replicate byte across V1 | |
| BLT cmp16 // Jump to the small string case if it's <32 bytes. | |
| CMP R4,$64,CR1 | |
| MOVD $16,R11 | |
| MOVD R3,R8 | |
| BLT CR1,cmp32 // Special case for length 32 - 63 | |
| MOVD $32,R12 | |
| MOVD $48,R6 | |
| RLDICR $0,R4,$63-6,R9 // R9 = len &^ 63 | |
| ADD R3,R9,R9 // R9 = &s[len &^ 63] | |
| ANDCC $63,R4 // (len &= 63) cmp 0. | |
| PCALIGN $16 | |
| loop64: | |
| LXVD2X (R0)(R8),V2 // Scan 64 bytes at a time, starting at &s[0] | |
| VCMPEQUBCC V2,V1,V6 | |
| BNE CR6,foundat0 // Match found at R8, jump out | |
| LXVD2X (R11)(R8),V2 | |
| VCMPEQUBCC V2,V1,V6 | |
| BNE CR6,foundat1 // Match found at R8+16 bytes, jump out | |
| LXVD2X (R12)(R8),V2 | |
| VCMPEQUBCC V2,V1,V6 | |
| BNE CR6,foundat2 // Match found at R8+32 bytes, jump out | |
| LXVD2X (R6)(R8),V2 | |
| VCMPEQUBCC V2,V1,V6 | |
| BNE CR6,foundat3 // Match found at R8+48 bytes, jump out | |
| ADD $64,R8 | |
| CMPU R8,R9,CR1 | |
| BNE CR1,loop64 // R8 != &s[len &^ 63]? | |
| PCALIGN $32 | |
| BEQ notfound // Is tail length 0? CR0 is set before entering loop64. | |
| CMP R4,$32 // Tail length >= 32, use cmp32 path. | |
| CMP R4,$16,CR1 | |
| BGE cmp32 | |
| ADD R8,R4,R9 | |
| ADD $-16,R9 | |
| BLE CR1,cmp64_tail_gt0 | |
| cmp64_tail_gt16: // Tail length 17 - 32 | |
| LXVD2X (R0)(R8),V2 | |
| VCMPEQUBCC V2,V1,V6 | |
| BNE CR6,foundat0 | |
| cmp64_tail_gt0: // Tail length 1 - 16 | |
| MOVD R9,R8 | |
| LXVD2X (R0)(R9),V2 | |
| VCMPEQUBCC V2,V1,V6 | |
| BNE CR6,foundat0 | |
| BR notfound | |
| cmp32: // Length 32 - 63 | |
| // Bytes 0 - 15 | |
| LXVD2X (R0)(R8),V2 | |
| VCMPEQUBCC V2,V1,V6 | |
| BNE CR6,foundat0 | |
| // Bytes 16 - 31 | |
| LXVD2X (R8)(R11),V2 | |
| VCMPEQUBCC V2,V1,V6 | |
| BNE CR6,foundat1 // Match found at R8+16 bytes, jump out | |
| BEQ notfound // Is length <= 32? (CR0 holds this comparison on entry to cmp32) | |
| CMP R4,$48 | |
| ADD R4,R8,R9 // Compute &s[len(s)-16] | |
| ADD $32,R8,R8 | |
| ADD $-16,R9,R9 | |
| ISEL CR0GT,R8,R9,R8 // R8 = len(s) <= 48 ? R9 : R8 | |
| // Bytes 33 - 47 | |
| LXVD2X (R0)(R8),V2 | |
| VCMPEQUBCC V2,V1,V6 | |
| BNE CR6,foundat0 // match found at R8+32 bytes, jump out | |
| BLE notfound | |
| // Bytes 48 - 63 | |
| MOVD R9,R8 // R9 holds the final check. | |
| LXVD2X (R0)(R9),V2 | |
| VCMPEQUBCC V2,V1,V6 | |
| BNE CR6,foundat0 // Match found at R8+48 bytes, jump out | |
| BR notfound | |
| // If ISA 3.0 instructions are unavailable, we need to account for the extra 16 added by CNTLZW. | |
| #ifndef GOPPC64_power9 | |
| #define ADJUST_FOR_CNTLZW -16 | |
| #else | |
| #define ADJUST_FOR_CNTLZW 0 | |
| #endif | |
| // Now, find the index of the 16B vector the match was discovered in. If CNTLZW is used | |
| // to determine the offset into the 16B vector, it will overcount by 16. Account for it here. | |
| foundat3: | |
| SUB R3,R8,R3 | |
| ADD $48+ADJUST_FOR_CNTLZW,R3 | |
| BR vfound | |
| foundat2: | |
| SUB R3,R8,R3 | |
| ADD $32+ADJUST_FOR_CNTLZW,R3 | |
| BR vfound | |
| foundat1: | |
| SUB R3,R8,R3 | |
| ADD $16+ADJUST_FOR_CNTLZW,R3 | |
| BR vfound | |
| foundat0: | |
| SUB R3,R8,R3 | |
| ADD $0+ADJUST_FOR_CNTLZW,R3 | |
| vfound: | |
| // Map equal values into a 16 bit value with earlier matches setting higher bits. | |
| #ifndef GOPPC64_power9 | |
| VBPERMQ V6,V0,V6 | |
| MFVRD V6,R4 | |
| CNTLZW R4,R4 | |
| #else | |
| #ifdef GOARCH_ppc64le | |
| // Put the value back into LE ordering by swapping doublewords. | |
| XXPERMDI V6,V6,$2,V6 | |
| #endif | |
| _VCZBEBB V6,R4 | |
| #endif | |
| ADD R3,R4,R3 | |
| RET | |
| cmp16: // Length 16 - 31 | |
| CMPU R4,$16 | |
| ADD R4,R3,R9 | |
| BLT cmp8 | |
| ADD $-16,R9,R9 // &s[len(s)-16] | |
| // Bytes 0 - 15 | |
| LXVD2X (R0)(R3),V2 | |
| VCMPEQUBCC V2,V1,V6 | |
| MOVD R3,R8 | |
| BNE CR6,foundat0 // Match found at R8+32 bytes, jump out | |
| BEQ notfound | |
| // Bytes 16 - 30 | |
| MOVD R9,R8 // R9 holds the final check. | |
| LXVD2X (R0)(R9),V2 | |
| VCMPEQUBCC V2,V1,V6 | |
| BNE CR6,foundat0 // Match found at R8+48 bytes, jump out | |
| BR notfound | |
| cmp8: // Length 8 - 15 | |
| #ifdef GOPPC64_power10 | |
| // Load all the bytes into a single VSR in BE order. | |
| SLD $56,R4,R5 | |
| LXVLL R3,R5,V2 | |
| // Compare and count the number which don't match. | |
| VCMPEQUB V2,V1,V6 | |
| VCLZLSBB V6,R3 | |
| // If count is the number of bytes, or more. No matches are found. | |
| CMPU R3,R4 | |
| MOVD $-1,R5 | |
| // Otherwise, the count is the index of the first match. | |
| ISEL CR0LT,R3,R5,R3 | |
| RET | |
| #else | |
| RLDIMI $8,R5,$48,R5 // Replicating the byte across the register. | |
| RLDIMI $16,R5,$32,R5 | |
| RLDIMI $32,R5,$0,R5 | |
| CMPU R4,$8 | |
| BLT cmp4 | |
| MOVD $-8,R11 | |
| ADD $-8,R4,R4 | |
| _LDBEX (R0)(R3),R10 | |
| _LDBEX (R11)(R9),R11 | |
| CMPB R10,R5,R10 | |
| CMPB R11,R5,R11 | |
| CMPU R10,$0 | |
| CMPU R11,$0,CR1 | |
| CNTLZD R10,R10 | |
| CNTLZD R11,R11 | |
| SRD $3,R10,R3 | |
| SRD $3,R11,R11 | |
| BNE found | |
| ADD R4,R11,R4 | |
| MOVD $-1,R3 | |
| ISEL CR1EQ,R3,R4,R3 | |
| RET | |
| cmp4: // Length 4 - 7 | |
| CMPU R4,$4 | |
| BLT cmp2 | |
| MOVD $-4,R11 | |
| ADD $-4,R4,R4 | |
| _LWBEX (R0)(R3),R10 | |
| _LWBEX (R11)(R9),R11 | |
| CMPB R10,R5,R10 | |
| CMPB R11,R5,R11 | |
| CNTLZW R10,R10 | |
| CNTLZW R11,R11 | |
| CMPU R10,$32 | |
| CMPU R11,$32,CR1 | |
| SRD $3,R10,R3 | |
| SRD $3,R11,R11 | |
| BNE found | |
| ADD R4,R11,R4 | |
| MOVD $-1,R3 | |
| ISEL CR1EQ,R3,R4,R3 | |
| RET | |
| cmp2: // Length 2 - 3 | |
| CMPU R4,$2 | |
| BLT cmp1 | |
| _LHBEX (R0)(R3),R10 | |
| CMPB R10,R5,R10 | |
| SLDCC $48,R10,R10 | |
| CNTLZD R10,R10 | |
| SRD $3,R10,R3 | |
| BNE found | |
| cmp1: // Length 1 | |
| MOVD $-1,R3 | |
| ANDCC $1,R4,R31 | |
| BEQ found | |
| MOVBZ -1(R9),R10 | |
| CMPB R10,R5,R10 | |
| ANDCC $1,R10 | |
| ADD $-1,R4 | |
| ISEL CR0EQ,R3,R4,R3 | |
| found: | |
| RET | |
| #endif | |
| notfound: | |
| MOVD $-1,R3 | |
| RET | |