blob: 26dabd9e6947497cfabcf1dd437c3c78043379ef [file] [log] [blame]
Shenghou Ma5f691cf2014-08-12 19:48:49 -04001// Copyright 2014 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
Russ Cox09d92b62014-12-05 19:13:20 -05005// +build ppc64 ppc64le
Shenghou Ma5f691cf2014-08-12 19:48:49 -04006
Austin Clements062e3542014-10-27 17:27:03 -04007#include "textflag.h"
Shenghou Ma5f691cf2014-08-12 19:48:49 -04008
9// void runtime·memmove(void*, void*, uintptr)
Michael Hudson-Doyle45c06b22015-10-09 12:44:27 +130010TEXT runtime·memmove(SB), NOSPLIT|NOFRAME, $0-24
Shenghou Ma5f691cf2014-08-12 19:48:49 -040011 MOVD to+0(FP), R3
12 MOVD from+8(FP), R4
13 MOVD n+16(FP), R5
Shenghou Ma5f691cf2014-08-12 19:48:49 -040014
Lynn Bogerc4807d42016-04-13 08:58:10 -050015 // Determine if there are doublewords to
16 // copy so a more efficient move can be done
Shenghou Ma5f691cf2014-08-12 19:48:49 -040017check:
Lynn Bogerc4807d42016-04-13 08:58:10 -050018 ANDCC $7, R5, R7 // R7: bytes to copy
19 SRAD $3, R5, R6 // R6: double words to copy
20 CMP R6, $0, CR1 // CR1[EQ] set if no double words to copy
Shenghou Ma5f691cf2014-08-12 19:48:49 -040021
Lynn Bogerc4807d42016-04-13 08:58:10 -050022 // Determine overlap by subtracting dest - src and comparing against the
23 // length. The catches the cases where src and dest are in different types
24 // of storage such as stack and static to avoid doing backward move when not
25 // necessary.
Michael Hudson-Doyle2c911142015-09-22 22:35:52 +120026
Lynn Bogerc4807d42016-04-13 08:58:10 -050027 SUB R4, R3, R8 // dest - src
28 CMPU R8, R5, CR2 // < len?
29 BC 12, 8, backward // BLT CR2 backward
30
31 // Copying forward if no overlap.
Michael Hudson-Doyle2c911142015-09-22 22:35:52 +120032
33 BC 12, 6, noforwardlarge // "BEQ CR1, noforwardlarge"
Lynn Bogerc4807d42016-04-13 08:58:10 -050034 MOVD R6,CTR // R6 = number of double words
35 SRADCC $2,R6,R8 // 32 byte chunks?
36 BNE forward32setup //
Michael Hudson-Doyle2c911142015-09-22 22:35:52 +120037
Lynn Bogerc4807d42016-04-13 08:58:10 -050038 // Move double words
Michael Hudson-Doyle2c911142015-09-22 22:35:52 +120039
Lynn Bogerc4807d42016-04-13 08:58:10 -050040forward8:
41 MOVD 0(R4), R8 // double word
42 ADD $8,R4
43 MOVD R8, 0(R3) //
44 ADD $8,R3
45 BC 16, 0, forward8
46 BR noforwardlarge // handle remainder
Michael Hudson-Doyle2c911142015-09-22 22:35:52 +120047
Lynn Bogerc4807d42016-04-13 08:58:10 -050048 // Prepare for moves of 32 bytes at a time.
Michael Hudson-Doyle2c911142015-09-22 22:35:52 +120049
Lynn Bogerc4807d42016-04-13 08:58:10 -050050forward32setup:
51 DCBTST (R3) // prepare data cache
52 DCBT (R4)
53 MOVD R8, CTR // double work count
54
55forward32:
56 MOVD 0(R4), R8 // load 4 double words
57 MOVD 8(R4), R9
58 MOVD 16(R4), R14
59 MOVD 24(R4), R15
60 ADD $32,R4
61 MOVD R8, 0(R3) // store those 4
62 MOVD R9, 8(R3)
63 MOVD R14,16(R3)
64 MOVD R15,24(R3)
65 ADD $32,R3 // bump up for next set
66 BC 16, 0, forward32 // continue
67 RLDCLCC $61,R5,$3,R6 // remaining doublewords
68 BEQ noforwardlarge
69 MOVD R6,CTR // set up the CTR
70 BR forward8
Michael Hudson-Doyle2c911142015-09-22 22:35:52 +120071
72noforwardlarge:
Lynn Bogerc4807d42016-04-13 08:58:10 -050073 CMP R7,$0 // any remaining bytes
74 BC 4, 1, LR
Michael Hudson-Doyle2c911142015-09-22 22:35:52 +120075
76forwardtail:
Lynn Bogerc4807d42016-04-13 08:58:10 -050077 MOVD R7, CTR // move tail bytes
Michael Hudson-Doyle2c911142015-09-22 22:35:52 +120078
79forwardtailloop:
Lynn Bogerc4807d42016-04-13 08:58:10 -050080 MOVBZ 0(R4), R8 // move single bytes
81 ADD $1,R4
82 MOVBZ R8, 0(R3)
83 ADD $1,R3
Michael Hudson-Doyle2c911142015-09-22 22:35:52 +120084 BC 16, 0, forwardtailloop
Austin Clements2774b372015-06-03 14:59:27 -040085 RET
Shenghou Ma5f691cf2014-08-12 19:48:49 -040086
87backward:
Lynn Bogerc4807d42016-04-13 08:58:10 -050088 // Copying backwards proceeds by copying R7 bytes then copying R6 double words.
Michael Hudson-Doyle2c911142015-09-22 22:35:52 +120089 // R3 and R4 are advanced to the end of the destination/source buffers
90 // respectively and moved back as we copy.
91
Lynn Bogerc4807d42016-04-13 08:58:10 -050092 ADD R5, R4, R4 // end of source
93 ADD R3, R5, R3 // end of dest
Michael Hudson-Doyle2c911142015-09-22 22:35:52 +120094
Lynn Bogerc4807d42016-04-13 08:58:10 -050095 BEQ nobackwardtail // earlier condition
Michael Hudson-Doyle2c911142015-09-22 22:35:52 +120096
Lynn Bogerc4807d42016-04-13 08:58:10 -050097 MOVD R7, CTR // bytes to move
Michael Hudson-Doyle2c911142015-09-22 22:35:52 +120098
99backwardtailloop:
Lynn Bogerc4807d42016-04-13 08:58:10 -0500100 MOVBZ -1(R4), R8 // point to last byte
101 SUB $1,R4
102 MOVBZ R8, -1(R3)
103 SUB $1,R3
Michael Hudson-Doyle2c911142015-09-22 22:35:52 +1200104 BC 16, 0, backwardtailloop
105
106nobackwardtail:
Lynn Bogerc4807d42016-04-13 08:58:10 -0500107 CMP R6,$0
108 BC 4, 5, LR
Michael Hudson-Doyle2c911142015-09-22 22:35:52 +1200109
110backwardlarge:
111 MOVD R6, CTR
112
113backwardlargeloop:
Lynn Bogerc4807d42016-04-13 08:58:10 -0500114 MOVD -8(R4), R8
115 SUB $8,R4
116 MOVD R8, -8(R3)
117 SUB $8,R3
118 BC 16, 0, backwardlargeloop //
Austin Clements2774b372015-06-03 14:59:27 -0400119 RET