// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !race

#include "../../../cmd/ld/textflag.h"

// Linux/ARM atomic operations.

// Because there is so much variation in ARM devices,
// the Linux kernel provides an appropriate compare-and-swap
// implementation at address 0xffff0fc0.  Caller sets:
//	R0 = old value
//	R1 = new value
//	R2 = addr
//	LR = return address
// The function returns with CS true if the swap happened.
// http://lxr.linux.no/linux+v2.6.37.2/arch/arm/kernel/entry-armv.S#L850
// On older kernels (before 2.6.24) the function can incorrectly
// report a conflict, so we have to double-check the compare ourselves
// and retry if necessary.
//
// http://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git;a=commit;h=b49c0f24cf6744a3f4fd09289fe7cade349dead5
//
TEXT cas<>(SB),NOSPLIT,$0
	MOVW	$0xffff0fc0, PC

TEXT ·CompareAndSwapInt32(SB),NOSPLIT,$0
	B	·CompareAndSwapUint32(SB)

// Implement using kernel cas for portability.
TEXT ·CompareAndSwapUint32(SB),NOSPLIT,$0-13
	MOVW	addr+0(FP), R2
	// trigger potential paging fault here,
	// because we don't know how to traceback through __kuser_cmpxchg
	MOVW	(R2), R0
	MOVW	old+4(FP), R0
casagain:
	MOVW	new+8(FP), R1
	BL	cas<>(SB)
	BCC	cascheck
	MOVW	$1, R0
casret:
	MOVB	R0, swapped+12(FP)
	RET
cascheck:
	// Kernel lies; double-check.
	MOVW	addr+0(FP), R2
	MOVW	old+4(FP), R0
	MOVW	0(R2), R3
	CMP	R0, R3
	BEQ	casagain
	MOVW	$0, R0
	B	casret

TEXT ·CompareAndSwapUintptr(SB),NOSPLIT,$0
	B	·CompareAndSwapUint32(SB)

TEXT ·CompareAndSwapPointer(SB),NOSPLIT,$0
	B	·CompareAndSwapUint32(SB)

TEXT ·AddInt32(SB),NOSPLIT,$0
	B	·AddUint32(SB)

// Implement using kernel cas for portability.
TEXT ·AddUint32(SB),NOSPLIT,$0-12
	MOVW	addr+0(FP), R2
	MOVW	delta+4(FP), R4
addloop1:
	MOVW	0(R2), R0
	MOVW	R0, R1
	ADD	R4, R1
	BL	cas<>(SB)
	BCC	addloop1
	MOVW	R1, new+8(FP)
	RET

TEXT ·AddUintptr(SB),NOSPLIT,$0
	B	·AddUint32(SB)

TEXT ·SwapInt32(SB),NOSPLIT,$0
	B	·SwapUint32(SB)

// Implement using kernel cas for portability.
TEXT ·SwapUint32(SB),NOSPLIT,$0-12
	MOVW	addr+0(FP), R2
	MOVW	new+4(FP), R1
swaploop1:
	MOVW	0(R2), R0
	MOVW	R0, R4 // cas smashes R0
	BL	cas<>(SB)
	BCC	swaploop1
	MOVW	R4, old+8(FP)
	RET

TEXT ·SwapUintptr(SB),NOSPLIT,$0
	B	·SwapUint32(SB)

TEXT ·SwapPointer(SB),NOSPLIT,$0
	B	·SwapUint32(SB)

TEXT cas64<>(SB),NOSPLIT,$0
	MOVW	$0xffff0f60, PC // __kuser_cmpxchg64: Linux-3.1 and above

TEXT kernelCAS64<>(SB),NOSPLIT,$0-21
	// int (*__kuser_cmpxchg64_t)(const int64_t *oldval, const int64_t *newval, volatile int64_t *ptr);
	MOVW	addr+0(FP), R2 // ptr
	// trigger potential paging fault here,
	// because we don't know how to traceback through __kuser_cmpxchg64
	MOVW	(R2), R0
	// make unaligned atomic access panic
	AND.S	$7, R2, R1
	BEQ 	2(PC)
	MOVW	R1, (R1)
	MOVW	$4(FP), R0 // oldval
	MOVW	$12(FP), R1 // newval
	BL	cas64<>(SB)
	MOVW.CS	$1, R0 // C is set if the kernel has changed *ptr
	MOVW.CC	$0, R0
	MOVW	R0, 20(FP)
	RET

TEXT ·generalCAS64(SB),NOSPLIT,$20-21
	// bool runtime·cas64(uint64 volatile *addr, uint64 old, uint64 new)
	MOVW	addr+0(FP), R0
	// trigger potential paging fault here,
	// because a fault in runtime.cas64 will hang.
	MOVW	(R0), R2
	// make unaligned atomic access panic
	AND.S	$7, R0, R1
	BEQ 	2(PC)
	MOVW	R1, (R1)
	MOVW	R0, 4(R13)
	MOVW	old_lo+4(FP), R1
	MOVW	R1, 8(R13)
	MOVW	old_hi+8(FP), R1
	MOVW	R1, 12(R13)
	MOVW	new_lo+12(FP), R2
	MOVW	R2, 16(R13)
	MOVW	new_hi+16(FP), R3
	MOVW	R3, 20(R13)
	BL  	runtime·cas64(SB)
	MOVB	R0, ret+20(FP)
	RET

GLOBL armCAS64(SB), $4

TEXT setupAndCallCAS64<>(SB),NOSPLIT,$-4-21
	MOVW	$0xffff0ffc, R0 // __kuser_helper_version
	MOVW	(R0), R0
	// __kuser_cmpxchg64 only present if helper version >= 5
	CMP 	$5, R0
	MOVW.CS	$kernelCAS64<>(SB), R1
	MOVW.CS	R1, armCAS64(SB)
	MOVW.CS	R1, PC
	MOVB	runtime·armArch(SB), R0
	// LDREXD, STREXD only present on ARMv6K or higher
	CMP	$6, R0 // TODO(minux): how to differentiate ARMv6 with ARMv6K?
	MOVW.CS	$·armCompareAndSwapUint64(SB), R1
	MOVW.CS	R1, armCAS64(SB)
	MOVW.CS	R1, PC
	// we are out of luck, can only use runtime's emulated 64-bit cas
	MOVW	$·generalCAS64(SB), R1
	MOVW	R1, armCAS64(SB)
	MOVW	R1, PC

TEXT ·CompareAndSwapInt64(SB),NOSPLIT,$0
	B   	·CompareAndSwapUint64(SB)

TEXT ·CompareAndSwapUint64(SB),NOSPLIT,$-4-21
	MOVW	armCAS64(SB), R0
	CMP 	$0, R0
	MOVW.NE	R0, PC
	B	setupAndCallCAS64<>(SB)

TEXT ·AddInt64(SB),NOSPLIT,$0
	B	·addUint64(SB)

TEXT ·AddUint64(SB),NOSPLIT,$0
	B	·addUint64(SB)

TEXT ·SwapInt64(SB),NOSPLIT,$0
	B	·swapUint64(SB)

TEXT ·SwapUint64(SB),NOSPLIT,$0
	B	·swapUint64(SB)

TEXT ·LoadInt32(SB),NOSPLIT,$0
	B	·LoadUint32(SB)

TEXT ·LoadUint32(SB),NOSPLIT,$0-8
	MOVW	addr+0(FP), R2
loadloop1:
	MOVW	0(R2), R0
	MOVW	R0, R1
	BL	cas<>(SB)
	BCC	loadloop1
	MOVW	R1, val+4(FP)
	RET

TEXT ·LoadInt64(SB),NOSPLIT,$0
	B	·loadUint64(SB)

TEXT ·LoadUint64(SB),NOSPLIT,$0
	B	·loadUint64(SB)

TEXT ·LoadUintptr(SB),NOSPLIT,$0
	B	·LoadUint32(SB)

TEXT ·LoadPointer(SB),NOSPLIT,$0
	B	·LoadUint32(SB)

TEXT ·StoreInt32(SB),NOSPLIT,$0
	B	·StoreUint32(SB)

TEXT ·StoreUint32(SB),NOSPLIT,$0-8
	MOVW	addr+0(FP), R2
	MOVW	val+4(FP), R1
storeloop1:
	MOVW	0(R2), R0
	BL	cas<>(SB)
	BCC	storeloop1
	RET

TEXT ·StoreInt64(SB),NOSPLIT,$0
	B	·storeUint64(SB)

TEXT ·StoreUint64(SB),NOSPLIT,$0
	B	·storeUint64(SB)

TEXT ·StoreUintptr(SB),NOSPLIT,$0
	B	·StoreUint32(SB)

TEXT ·StorePointer(SB),NOSPLIT,$0
	B	·StoreUint32(SB)