// Copyright 2015 The Go Authors. All rights reserved.

// Use of this source code is governed by a BSD-style

// license that can be found in the LICENSE file.

#include "go_asm.h"

#include "textflag.h"

#include "funcdata.h"

// func armcas(ptr *int32, old, new int32) bool

// Atomically:

// if *ptr == old {

// *ptr = new

// return true

// } else {

// return false

// }

//

// To implement ·cas in sys_$GOOS_arm.s

// using the native instructions, use:

//

// TEXT ·cas(SB),NOSPLIT,$0

// B ·armcas(SB)

//

TEXT ·armcas(SB),NOSPLIT,$0-13

MOVW ptr+0(FP), R1

MOVW old+4(FP), R2

MOVW new+8(FP), R3

casl:

LDREX (R1), R0

CMP R0, R2

BNE casfail

#ifndef GOARM_7

MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11

CMP $0, R11

BEQ 2(PC)

#endif

DMB MB_ISHST

STREX R3, (R1), R0

CMP $0, R0

BNE casl

MOVW $1, R0

#ifndef GOARM_7

CMP $0, R11

BEQ 2(PC)

#endif

DMB MB_ISH

MOVB R0, ret+12(FP)

RET

casfail:

MOVW $0, R0

MOVB R0, ret+12(FP)

RET

// stubs

TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-8

B ·Load(SB)

TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-8

B ·Load(SB)

TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-8

B ·Load(SB)

TEXT ·Casint32(SB),NOSPLIT,$0-13

B ·Cas(SB)

TEXT ·Casint64(SB),NOSPLIT,$-4-21

B ·Cas64(SB)

TEXT ·Casuintptr(SB),NOSPLIT,$0-13

B ·Cas(SB)

TEXT ·Casp1(SB),NOSPLIT,$0-13

B ·Cas(SB)

TEXT ·CasRel(SB),NOSPLIT,$0-13

B ·Cas(SB)

TEXT ·Loadint32(SB),NOSPLIT,$0-8

B ·Load(SB)

TEXT ·Loadint64(SB),NOSPLIT,$-4-12

B ·Load64(SB)

TEXT ·Loaduintptr(SB),NOSPLIT,$0-8

B ·Load(SB)

TEXT ·Loaduint(SB),NOSPLIT,$0-8

B ·Load(SB)

TEXT ·Storeint32(SB),NOSPLIT,$0-8

B ·Store(SB)

TEXT ·Storeint64(SB),NOSPLIT,$0-12

B ·Store64(SB)

TEXT ·Storeuintptr(SB),NOSPLIT,$0-8

B ·Store(SB)

TEXT ·StorepNoWB(SB),NOSPLIT,$0-8

B ·Store(SB)

TEXT ·StoreRel(SB),NOSPLIT,$0-8

B ·Store(SB)

TEXT ·StoreReluintptr(SB),NOSPLIT,$0-8

B ·Store(SB)

TEXT ·Xaddint32(SB),NOSPLIT,$0-12

B ·Xadd(SB)

TEXT ·Xaddint64(SB),NOSPLIT,$-4-20

B ·Xadd64(SB)

TEXT ·Xadduintptr(SB),NOSPLIT,$0-12

B ·Xadd(SB)

TEXT ·Xchgint32(SB),NOSPLIT,$0-12

B ·Xchg(SB)

TEXT ·Xchgint64(SB),NOSPLIT,$-4-20

B ·Xchg64(SB)

// 64-bit atomics

// The native ARM implementations use LDREXD/STREXD, which are

// available on ARMv6k or later. We use them only on ARMv7.

// On older ARM, we use Go implementations which simulate 64-bit

// atomics with locks.

TEXT armCas64<>(SB),NOSPLIT,$0-21

// addr is already in R1

MOVW old_lo+4(FP), R2

MOVW old_hi+8(FP), R3

MOVW new_lo+12(FP), R4

MOVW new_hi+16(FP), R5

cas64loop:

LDREXD (R1), R6 // loads R6 and R7

CMP R2, R6

BNE cas64fail

CMP R3, R7

BNE cas64fail

DMB MB_ISHST

STREXD R4, (R1), R0 // stores R4 and R5

CMP $0, R0

BNE cas64loop

MOVW $1, R0

DMB MB_ISH

MOVBU R0, swapped+20(FP)

RET

cas64fail:

MOVW $0, R0

MOVBU R0, swapped+20(FP)

RET

TEXT armXadd64<>(SB),NOSPLIT,$0-20

// addr is already in R1

MOVW delta_lo+4(FP), R2

MOVW delta_hi+8(FP), R3

add64loop:

LDREXD (R1), R4 // loads R4 and R5

ADD.S R2, R4

ADC R3, R5

DMB MB_ISHST

STREXD R4, (R1), R0 // stores R4 and R5

CMP $0, R0

BNE add64loop

DMB MB_ISH

MOVW R4, new_lo+12(FP)

MOVW R5, new_hi+16(FP)

RET

TEXT armXchg64<>(SB),NOSPLIT,$0-20

// addr is already in R1

MOVW new_lo+4(FP), R2

MOVW new_hi+8(FP), R3

swap64loop:

LDREXD (R1), R4 // loads R4 and R5

DMB MB_ISHST

STREXD R2, (R1), R0 // stores R2 and R3

CMP $0, R0

BNE swap64loop

DMB MB_ISH

MOVW R4, old_lo+12(FP)

MOVW R5, old_hi+16(FP)

RET

TEXT armLoad64<>(SB),NOSPLIT,$0-12

// addr is already in R1

LDREXD (R1), R2 // loads R2 and R3

DMB MB_ISH

MOVW R2, val_lo+4(FP)

MOVW R3, val_hi+8(FP)

RET

TEXT armStore64<>(SB),NOSPLIT,$0-12

// addr is already in R1

MOVW val_lo+4(FP), R2

MOVW val_hi+8(FP), R3

store64loop:

LDREXD (R1), R4 // loads R4 and R5

DMB MB_ISHST

STREXD R2, (R1), R0 // stores R2 and R3

CMP $0, R0

BNE store64loop

DMB MB_ISH

RET

TEXT armAnd8<>(SB),NOSPLIT,$0-5

// addr is already in R1

MOVB v+4(FP), R2

and8loop:

LDREXB (R1), R6

DMB MB_ISHST

AND R2, R6

STREXB R6, (R1), R0

CMP $0, R0

BNE and8loop

DMB MB_ISH

RET

TEXT armOr8<>(SB),NOSPLIT,$0-5

// addr is already in R1

MOVB v+4(FP), R2

or8loop:

LDREXB (R1), R6

DMB MB_ISHST

ORR R2, R6

STREXB R6, (R1), R0

CMP $0, R0

BNE or8loop

DMB MB_ISH

RET

TEXT armXchg8<>(SB),NOSPLIT,$0-9

// addr is already in R1

MOVB v+4(FP), R2

xchg8loop:

LDREXB (R1), R6

DMB MB_ISHST

STREXB R2, (R1), R0

CMP $0, R0

BNE xchg8loop

DMB MB_ISH

MOVB R6, ret+8(FP)

RET

// The following functions all panic if their address argument isn't

// 8-byte aligned. Since we're calling back into Go code to do this,

// we have to cooperate with stack unwinding. In the normal case, the

// functions tail-call into the appropriate implementation, which

// means they must not open a frame. Hence, when they go down the

// panic path, at that point they push the LR to create a real frame

// (they don't need to pop it because panic won't return; however, we

// do need to set the SP delta back).

// Check if R1 is 8-byte aligned, panic if not.

// Clobbers R2.

#define CHECK_ALIGN \

AND.S $7, R1, R2 \

BEQ 4(PC) \

MOVW.W R14, -4(R13) /* prepare a real frame */ \

BL ·panicUnaligned(SB) \

ADD $4, R13 /* compensate SP delta */

TEXT ·Cas64(SB),NOSPLIT,$-4-21

NO_LOCAL_POINTERS

MOVW addr+0(FP), R1

CHECK_ALIGN

#ifndef GOARM_7

MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11

CMP $1, R11

BEQ 2(PC)

JMP ·goCas64(SB)

#endif

JMP armCas64<>(SB)

TEXT ·Xadd64(SB),NOSPLIT,$-4-20

NO_LOCAL_POINTERS

MOVW addr+0(FP), R1

CHECK_ALIGN

#ifndef GOARM_7

MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11

CMP $1, R11

BEQ 2(PC)

JMP ·goXadd64(SB)

#endif

JMP armXadd64<>(SB)

TEXT ·Xchg64(SB),NOSPLIT,$-4-20

NO_LOCAL_POINTERS

MOVW addr+0(FP), R1

CHECK_ALIGN

#ifndef GOARM_7

MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11

CMP $1, R11

BEQ 2(PC)

JMP ·goXchg64(SB)

#endif

JMP armXchg64<>(SB)

TEXT ·Load64(SB),NOSPLIT,$-4-12

NO_LOCAL_POINTERS

MOVW addr+0(FP), R1

CHECK_ALIGN

#ifndef GOARM_7

MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11

CMP $1, R11

BEQ 2(PC)

JMP ·goLoad64(SB)

#endif

JMP armLoad64<>(SB)

TEXT ·Store64(SB),NOSPLIT,$-4-12

NO_LOCAL_POINTERS

MOVW addr+0(FP), R1

CHECK_ALIGN

#ifndef GOARM_7

MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11

CMP $1, R11

BEQ 2(PC)

JMP ·goStore64(SB)

#endif

JMP armStore64<>(SB)

TEXT ·And8(SB),NOSPLIT,$-4-5

NO_LOCAL_POINTERS

MOVW addr+0(FP), R1

// Uses STREXB/LDREXB that is armv6k or later.

// For simplicity we only enable this on armv7.

#ifndef GOARM_7

MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11

CMP $1, R11

BEQ 2(PC)

JMP ·goAnd8(SB)

#endif

JMP armAnd8<>(SB)

TEXT ·Or8(SB),NOSPLIT,$-4-5

NO_LOCAL_POINTERS

MOVW addr+0(FP), R1

// Uses STREXB/LDREXB that is armv6k or later.

// For simplicity we only enable this on armv7.

#ifndef GOARM_7

MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11

CMP $1, R11

BEQ 2(PC)

JMP ·goOr8(SB)

#endif

JMP armOr8<>(SB)

TEXT ·Xchg8(SB),NOSPLIT,$-4-9

NO_LOCAL_POINTERS

MOVW addr+0(FP), R1

// Uses STREXB/LDREXB that is armv6k or later.

// For simplicity we only enable this on armv7.

#ifndef GOARM_7

MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11

CMP $1, R11

BEQ 2(PC)

JMP ·goXchg8(SB)

#endif

JMP armXchg8<>(SB)