blob: dbb3b90fcfeafad84b7d675dcdf405b1438c2151 [file] [log] [blame]
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build ppc64 ppc64le
#include "textflag.h"
// See memmove Go doc for important implementation constraints.
// func memmove(to, from unsafe.Pointer, n uintptr)
TEXT runtime·memmove(SB), NOSPLIT|NOFRAME, $0-24
MOVD to+0(FP), R3
MOVD from+8(FP), R4
MOVD n+16(FP), R5
// Determine if there are doublewords to
// copy so a more efficient move can be done
check:
ANDCC $7, R5, R7 // R7: bytes to copy
SRD $3, R5, R6 // R6: double words to copy
CMP R6, $0, CR1 // CR1[EQ] set if no double words to copy
// Determine overlap by subtracting dest - src and comparing against the
// length. The catches the cases where src and dest are in different types
// of storage such as stack and static to avoid doing backward move when not
// necessary.
SUB R4, R3, R8 // dest - src
CMPU R8, R5, CR2 // < len?
BC 12, 8, backward // BLT CR2 backward
// Copying forward if no overlap.
BC 12, 6, noforwardlarge // "BEQ CR1, noforwardlarge"
SRDCC $2,R6,R8 // 32 byte chunks?
BNE forward32setup //
MOVD R6,CTR // R6 = number of double words
// Move double words
forward8:
MOVD 0(R4), R8 // double word
ADD $8,R4
MOVD R8, 0(R3) //
ADD $8,R3
BC 16, 0, forward8
BR noforwardlarge // handle remainder
// Prepare for moves of 32 bytes at a time.
forward32setup:
DCBTST (R3) // prepare data cache
DCBT (R4)
MOVD R8, CTR // double work count
MOVD $16, R8
forward32:
LXVD2X (R4+R0), VS32 // load 16 bytes
LXVD2X (R4+R8), VS33
ADD $32, R4
STXVD2X VS32, (R3+R0) // store 16 bytes
STXVD2X VS33, (R3+R8)
ADD $32,R3 // bump up for next set
BC 16, 0, forward32 // continue
RLDCLCC $61,R5,$3,R6 // remaining doublewords
BEQ noforwardlarge
MOVD R6,CTR // set up the CTR
BR forward8
noforwardlarge:
CMP R7,$0 // any remaining bytes
BC 4, 1, LR // ble lr
forwardtail:
MOVD R7, CTR // move tail bytes
forwardtailloop:
MOVBZ 0(R4), R8 // move single bytes
ADD $1,R4
MOVBZ R8, 0(R3)
ADD $1,R3
BC 16, 0, forwardtailloop
RET
backward:
// Copying backwards proceeds by copying R7 bytes then copying R6 double words.
// R3 and R4 are advanced to the end of the destination/source buffers
// respectively and moved back as we copy.
ADD R5, R4, R4 // end of source
ADD R3, R5, R3 // end of dest
BEQ nobackwardtail // earlier condition
MOVD R7, CTR // bytes to move
backwardtailloop:
MOVBZ -1(R4), R8 // point to last byte
SUB $1,R4
MOVBZ R8, -1(R3)
SUB $1,R3
BC 16, 0, backwardtailloop // bndz
nobackwardtail:
BC 4, 5, LR // ble CR1 lr
backwardlarge:
MOVD R6, CTR
SUB R3, R4, R9 // Use vsx if moving
CMP R9, $32 // at least 32 byte chunks
BLT backwardlargeloop // and distance >= 32
SRDCC $2,R6,R8 // 32 byte chunks
BNE backward32setup
backwardlargeloop:
MOVD -8(R4), R8
SUB $8,R4
MOVD R8, -8(R3)
SUB $8,R3
BC 16, 0, backwardlargeloop // bndz
RET
backward32setup:
MOVD R8, CTR // set up loop ctr
MOVD $16, R8 // 32 bytes at at time
backward32loop:
SUB $32, R4
SUB $32, R3
LXVD2X (R4+R0), VS32 // load 16 bytes
LXVD2X (R4+R8), VS33
STXVD2X VS32, (R3+R0) // store 16 bytes
STXVD2X VS33, (R3+R8)
BC 16, 0, backward32loop // bndz
BC 4, 5, LR // ble CR1 lr
MOVD R6, CTR
BR backwardlargeloop