From db189d02ef150517e955d0d7a913168dd48f21cd Mon Sep 17 00:00:00 2001
From: David Faust <david.faust@oracle.com>
Date: Mon, 24 Jul 2023 09:45:17 -0700
Subject: [PATCH] bpf: add v3 atomic instructions
This patch adds support for the general atomic operations introduced in
eBPF v3. In addition to the existing atomic add instruction, this adds:
- Atomic and, or, xor
- Fetching versions of these operations (including add)
- Atomic exchange
- Atomic compare-and-exchange
To control emission of these instructions, a new target option
-m[no-]v3-atomics is added. This option is enabled by -mcpu=v3
and above.
Support for these instructions was recently added in binutils.
gcc/
* config/bpf/bpf.opt (mv3-atomics): New option.
* config/bpf/bpf.cc (bpf_option_override): Handle it here.
* config/bpf/bpf.h (enum_reg_class): Add R0 class.
(REG_CLASS_NAMES): Likewise.
(REG_CLASS_CONTENTS): Likewise.
(REGNO_REG_CLASS): Handle R0.
* config/bpf/bpf.md (UNSPEC_XADD): Rename to UNSPEC_AADD.
(UNSPEC_AAND): New unspec.
(UNSPEC_AOR): Likewise.
(UNSPEC_AXOR): Likewise.
(UNSPEC_AFADD): Likewise.
(UNSPEC_AFAND): Likewise.
(UNSPEC_AFOR): Likewise.
(UNSPEC_AFXOR): Likewise.
(UNSPEC_AXCHG): Likewise.
(UNSPEC_ACMPX): Likewise.
(atomic_add<mode>): Use UNSPEC_AADD and atomic type attribute.
Move to...
* config/bpf/atomic.md: ...Here. New file.
* config/bpf/constraints.md (t): New constraint for R0.
* doc/invoke.texi (eBPF Options): Document -mv3-atomics.
gcc/testsuite/
* gcc.target/bpf/atomic-cmpxchg-1.c: New test.
* gcc.target/bpf/atomic-cmpxchg-2.c: New test.
* gcc.target/bpf/atomic-fetch-op-1.c: New test.
* gcc.target/bpf/atomic-fetch-op-2.c: New test.
* gcc.target/bpf/atomic-fetch-op-3.c: New test.
* gcc.target/bpf/atomic-op-1.c: New test.
* gcc.target/bpf/atomic-op-2.c: New test.
* gcc.target/bpf/atomic-op-3.c: New test.
* gcc.target/bpf/atomic-xchg-1.c: New test.
* gcc.target/bpf/atomic-xchg-2.c: New test.
---
gcc/config/bpf/atomic.md | 185 ++++++++++++++++++
gcc/config/bpf/bpf.cc | 3 +
gcc/config/bpf/bpf.h | 6 +-
gcc/config/bpf/bpf.md | 29 ++-
gcc/config/bpf/bpf.opt | 4 +
gcc/config/bpf/constraints.md | 3 +
gcc/doc/invoke.texi | 8 +-
.../gcc.target/bpf/atomic-cmpxchg-1.c | 19 ++
.../gcc.target/bpf/atomic-cmpxchg-2.c | 19 ++
.../gcc.target/bpf/atomic-fetch-op-1.c | 50 +++++
.../gcc.target/bpf/atomic-fetch-op-2.c | 50 +++++
.../gcc.target/bpf/atomic-fetch-op-3.c | 49 +++++
gcc/testsuite/gcc.target/bpf/atomic-op-1.c | 49 +++++
gcc/testsuite/gcc.target/bpf/atomic-op-2.c | 49 +++++
gcc/testsuite/gcc.target/bpf/atomic-op-3.c | 49 +++++
gcc/testsuite/gcc.target/bpf/atomic-xchg-1.c | 20 ++
gcc/testsuite/gcc.target/bpf/atomic-xchg-2.c | 20 ++
17 files changed, 593 insertions(+), 19 deletions(-)
create mode 100644 gcc/config/bpf/atomic.md
create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-cmpxchg-1.c
create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-cmpxchg-2.c
create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-op-1.c
create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-op-2.c
create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-op-3.c
create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-op-1.c
create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-op-2.c
create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-op-3.c
create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-xchg-1.c
create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-xchg-2.c
diff --git a/gcc/config/bpf/atomic.md b/gcc/config/bpf/atomic.md
new file mode 100644
index 000000000000..caf8cc15cd4f
--- /dev/null
+++ b/gcc/config/bpf/atomic.md
@@ -0,0 +1,185 @@
+;; Machine description for eBPF.
+;; Copyright (C) 2023 Free Software Foundation, Inc.
+
+;; This file is part of GCC.
+
+;; GCC is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation; either version 3, or (at your option)
+;; any later version.
+
+;; GCC is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GCC; see the file COPYING3. If not see
+;; <http://www.gnu.org/licenses/>.
+
+
+(define_mode_iterator AMO [SI DI])
+
+;;; Plain atomic modify operations.
+
+;; Non-fetching atomic add predates all other BPF atomic insns.
+;; Use xadd{w,dw} for compatibility with older GAS without support
+;; for v3 atomics. Newer GAS supports "aadd[32]" in line with the
+;; other atomic operations.
+(define_insn "atomic_add<AMO:mode>"
+ [(set (match_operand:AMO 0 "memory_operand" "+m")
+ (unspec_volatile:AMO
+ [(plus:AMO (match_dup 0)
+ (match_operand:AMO 1 "register_operand" "r"))
+ (match_operand:SI 2 "const_int_operand")] ;; Memory model.
+ UNSPEC_AADD))]
+ ""
+ "{xadd<mop>\t%0,%1|lock *(<smop> *)(%w0) += %w1}"
+ [(set_attr "type" "atomic")])
+
+(define_insn "atomic_and<AMO:mode>"
+ [(set (match_operand:AMO 0 "memory_operand" "+m")
+ (unspec_volatile:AMO
+ [(and:AMO (match_dup 0)
+ (match_operand:AMO 1 "register_operand" "r"))
+ (match_operand:SI 2 "const_int_operand")] ;; Memory model.
+ UNSPEC_AAND))]
+ "bpf_has_v3_atomics"
+ "{aand<msuffix>\t%0,%1|lock *(<smop> *)(%w0) &= %w1}")
+
+(define_insn "atomic_or<AMO:mode>"
+ [(set (match_operand:AMO 0 "memory_operand" "+m")
+ (unspec_volatile:AMO
+ [(ior:AMO (match_dup 0)
+ (match_operand:AMO 1 "register_operand" "r"))
+ (match_operand:SI 2 "const_int_operand")] ;; Memory model.
+ UNSPEC_AOR))]
+ "bpf_has_v3_atomics"
+ "{aor<msuffix>\t%0,%1|lock *(<smop> *)(%w0) %|= %w1}")
+
+(define_insn "atomic_xor<AMO:mode>"
+ [(set (match_operand:AMO 0 "memory_operand" "+m")
+ (unspec_volatile:AMO
+ [(xor:AMO (match_dup 0)
+ (match_operand:AMO 1 "register_operand" "r"))
+ (match_operand:SI 2 "const_int_operand")] ;; Memory model.
+ UNSPEC_AXOR))]
+ "bpf_has_v3_atomics"
+ "{axor<msuffix>\t%0,%1|lock *(<smop> *)(%w0) ^= %w1}")
+
+;;; Feching (read-modify-store) versions of atomic operations.
+
+(define_insn "atomic_fetch_add<AMO:mode>"
+ [(set (match_operand:AMO 0 "register_operand" "=r") ; output
+ (match_operand:AMO 1 "memory_operand" "+m"))
+ (set (match_dup 1)
+ (unspec_volatile:AMO
+ [(plus:AMO (match_dup 1)
+ (match_operand:AMO 2 "nonmemory_operand" "0")) ; second operand to op
+ (match_operand:AMO 3 "const_int_operand")] ;; Memory model
+ UNSPEC_AFADD))]
+ "bpf_has_v3_atomics"
+ "{afadd<msuffix>\t%1,%0|%w0 = atomic_fetch_add((<smop> *)(%1), %w0)}")
+
+(define_insn "atomic_fetch_and<AMO:mode>"
+ [(set (match_operand:AMO 0 "register_operand" "=r")
+ (match_operand:AMO 1 "memory_operand" "+m"))
+ (set (match_dup 1)
+ (unspec_volatile:AMO
+ [(and:AMO (match_dup 1)
+ (match_operand:AMO 2 "nonmemory_operand" "0"))
+ (match_operand:AMO 3 "const_int_operand")]
+ UNSPEC_AFAND))]
+ "bpf_has_v3_atomics"
+ "{afand<msuffix>\t%1,%0|%w0 = atomic_fetch_and((<smop> *)(%1), %w0)}")
+
+(define_insn "atomic_fetch_or<AMO:mode>"
+ [(set (match_operand:AMO 0 "register_operand" "=r")
+ (match_operand:AMO 1 "memory_operand" "+m"))
+ (set (match_dup 1)
+ (unspec_volatile:AMO
+ [(ior:AMO (match_dup 1)
+ (match_operand:AMO 2 "nonmemory_operand" "0"))
+ (match_operand:AMO 3 "const_int_operand")]
+ UNSPEC_AFOR))]
+ "bpf_has_v3_atomics"
+ "{afor<msuffix>\t%1,%0|%w0 = atomic_fetch_or((<smop> *)(%1), %w0)}")
+
+(define_insn "atomic_fetch_xor<AMO:mode>"
+ [(set (match_operand:AMO 0 "register_operand" "=r")
+ (match_operand:AMO 1 "memory_operand" "+m"))
+ (set (match_dup 1)
+ (unspec_volatile:AMO
+ [(xor:AMO (match_dup 1)
+ (match_operand:AMO 2 "nonmemory_operand" "0"))
+ (match_operand:AMO 3 "const_int_operand")]
+ UNSPEC_AFXOR))]
+ "bpf_has_v3_atomics"
+ "{afxor<msuffix>\t%1,%0|%w0 = atomic_fetch_xor((<smop> *)(%1), %w0)}")
+
+;; Weird suffixes used in pseudo-c atomic compare-exchange insns.
+(define_mode_attr pcaxsuffix [(SI "32_32") (DI "_64")])
+
+(define_insn "atomic_exchange<AMO:mode>"
+ [(set (match_operand:AMO 0 "register_operand" "=r")
+ (unspec_volatile:AMO
+ [(match_operand:AMO 1 "memory_operand" "+m")
+ (match_operand:AMO 3 "const_int_operand")]
+ UNSPEC_AXCHG))
+ (set (match_dup 1)
+ (match_operand:AMO 2 "nonmemory_operand" "0"))]
+ "bpf_has_v3_atomics"
+ "{axchg<msuffix>\t%1,%0|%w0 = xchg<pcaxsuffix>(%1, %w0)}")
+
+;; The eBPF atomic-compare-and-exchange instruction has the form
+;; acmp [%dst+offset], %src
+;; The instruction atomically compares the value addressed by %dst+offset
+;; with register R0. If they match, the value at %dst+offset is overwritten
+;; with the value of %src. Otherwise, no write occurs. In either case, the
+;; original value of %dst+offset is zero-extended and loaded back into R0.
+
+(define_expand "atomic_compare_and_swap<AMO:mode>"
+ [(match_operand:SI 0 "register_operand" "=r") ;; bool success
+ (match_operand:AMO 1 "register_operand" "=r") ;; old value
+ (match_operand:AMO 2 "memory_operand" "+m") ;; memory
+ (match_operand:AMO 3 "register_operand") ;; expected
+ (match_operand:AMO 4 "register_operand") ;; desired
+ (match_operand:SI 5 "const_int_operand") ;; is_weak (unused)
+ (match_operand:SI 6 "const_int_operand") ;; success model (unused)
+ (match_operand:SI 7 "const_int_operand")] ;; failure model (unused)
+ "bpf_has_v3_atomics"
+{
+ /* Load the expected value (into R0 by constraint of below). */
+ emit_move_insn (operands[1], operands[3]);
+
+ /* Emit the acmp. */
+ emit_insn (gen_atomic_compare_and_swap<AMO:mode>_1 (operands[1], operands[2], operands[3], operands[4]));
+
+ /* Assume that the operation was successful. */
+ emit_move_insn (operands[0], const1_rtx);
+ rtx_code_label *success_label = gen_label_rtx ();
+
+ /* Compare value that was in memory (now in R0/op[1]) to expected value.
+ If they are equal, then the write occurred. Otherwise, indicate fail in output. */
+ emit_cmp_and_jump_insns (operands[1], operands[3], EQ, 0,
+ GET_MODE (operands[1]), 1, success_label);
+ emit_move_insn (operands[0], const0_rtx);
+
+ if (success_label)
+ {
+ emit_label (success_label);
+ LABEL_NUSES (success_label) = 1;
+ }
+ DONE;
+})
+
+(define_insn "atomic_compare_and_swap<AMO:mode>_1"
+ [(set (match_operand:AMO 0 "register_operand" "+t") ;; R0 is both input (expected value)
+ (unspec_volatile:AMO ;; and output (original value)
+ [(match_dup 0) ;; result depends on R0
+ (match_operand:AMO 1 "memory_operand") ;; memory
+ (match_operand:AMO 2 "register_operand") ;; expected
+ (match_operand:AMO 3 "register_operand")] ;; desired
+ UNSPEC_ACMP))]
+ "bpf_has_v3_atomics"
+ "{acmp<msuffix>\t%1,%3|%w0 = cmpxchg<pcaxsuffix>(%1, %w0, %w3)}")
diff --git a/gcc/config/bpf/bpf.cc b/gcc/config/bpf/bpf.cc
index 2e1e3e3abcf2..0e07b416add8 100644
--- a/gcc/config/bpf/bpf.cc
+++ b/gcc/config/bpf/bpf.cc
@@ -253,6 +253,9 @@ bpf_option_override (void)
if (bpf_has_jmp32 == -1)
bpf_has_jmp32 = (bpf_isa >= ISA_V3);
+ if (bpf_has_v3_atomics == -1)
+ bpf_has_v3_atomics = (bpf_isa >= ISA_V3);
+
if (bpf_has_bswap == -1)
bpf_has_bswap = (bpf_isa >= ISA_V4);
diff --git a/gcc/config/bpf/bpf.h b/gcc/config/bpf/bpf.h
index 9561bf59b800..ccba7f8b3334 100644
--- a/gcc/config/bpf/bpf.h
+++ b/gcc/config/bpf/bpf.h
@@ -177,6 +177,7 @@
enum reg_class
{
NO_REGS, /* no registers in set. */
+ R0, /* register r0. */
ALL_REGS, /* all registers. */
LIM_REG_CLASSES /* max value + 1. */
};
@@ -190,6 +191,7 @@ enum reg_class
#define REG_CLASS_NAMES \
{ \
"NO_REGS", \
+ "R0", \
"ALL_REGS" \
}
@@ -203,6 +205,7 @@ enum reg_class
#define REG_CLASS_CONTENTS \
{ \
0x00000000, /* NO_REGS */ \
+ 0x00000001, /* R0 */ \
0x00000fff, /* ALL_REGS */ \
}
@@ -210,7 +213,8 @@ enum reg_class
register REGNO. In general there is more that one such class;
choose a class which is "minimal", meaning that no smaller class
also contains the register. */
-#define REGNO_REG_CLASS(REGNO) ((void)(REGNO), GENERAL_REGS)
+#define REGNO_REG_CLASS(REGNO) \
+ ((REGNO) == 0 ? R0 : GENERAL_REGS)
/* A macro whose definition is the name of the class to which a
valid base register must belong. A base register is one used in
diff --git a/gcc/config/bpf/bpf.md b/gcc/config/bpf/bpf.md
index 1b5e1900d4fa..2ffc4ebd17e9 100644
--- a/gcc/config/bpf/bpf.md
+++ b/gcc/config/bpf/bpf.md
@@ -35,7 +35,16 @@
(define_c_enum "unspec" [
UNSPEC_LDINDABS
- UNSPEC_XADD
+ UNSPEC_AADD
+ UNSPEC_AAND
+ UNSPEC_AOR
+ UNSPEC_AXOR
+ UNSPEC_AFADD
+ UNSPEC_AFAND
+ UNSPEC_AFOR
+ UNSPEC_AFXOR
+ UNSPEC_AXCHG
+ UNSPEC_ACMP
])
;;;; Constants
@@ -67,11 +76,10 @@
;; st generic store instructions for immediates.
;; stx generic store instructions.
;; jmp jump instructions.
-;; xadd atomic exchange-and-add instructions.
;; multi multiword sequence (or user asm statements).
(define_attr "type"
- "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,xadd,multi"
+ "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,multi,atomic"
(const_string "unknown"))
;; Length of instruction in bytes.
@@ -548,17 +556,4 @@
"{ldabs<ldop>\t%0|r0 = *(<pldop> *) skb[%0]}"
[(set_attr "type" "ld")])
-;;;; Atomic increments
-
-(define_mode_iterator AMO [SI DI])
-
-(define_insn "atomic_add<AMO:mode>"
- [(set (match_operand:AMO 0 "memory_operand" "+m")
- (unspec_volatile:AMO
- [(plus:AMO (match_dup 0)
- (match_operand:AMO 1 "register_operand" "r"))
- (match_operand:SI 2 "const_int_operand")] ;; Memory model.
- UNSPEC_XADD))]
- ""
- "{xadd<mop>\t%0,%1|*(<smop> *) %0 += %1}"
- [(set_attr "type" "xadd")])
+(include "atomic.md")
diff --git a/gcc/config/bpf/bpf.opt b/gcc/config/bpf/bpf.opt
index bd35f8dbd0ce..b21cfcab9ea7 100644
--- a/gcc/config/bpf/bpf.opt
+++ b/gcc/config/bpf/bpf.opt
@@ -59,6 +59,10 @@ mjmp32
Target Var(bpf_has_jmp32) Init(-1)
Enable 32-bit jump instructions.
+mv3-atomics
+Target Var(bpf_has_v3_atomics) Init(-1)
+Enable general atomic operations introduced in v3 ISA.
+
mbswap
Target Var(bpf_has_bswap) Init(-1)
Enable byte swap instructions.
diff --git a/gcc/config/bpf/constraints.md b/gcc/config/bpf/constraints.md
index 33f9177b8eb1..199dd00c0cb7 100644
--- a/gcc/config/bpf/constraints.md
+++ b/gcc/config/bpf/constraints.md
@@ -30,6 +30,9 @@
"A constant call address."
(match_code "const,symbol_ref,label_ref,const_int"))
+(define_register_constraint "t" "R0"
+ "Register r0")
+
;;
;; Memory constraints.
;;
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index fa765d5a0dd9..e0fd7bd5b72e 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -947,7 +947,7 @@ Objective-C and Objective-C++ Dialects}.
@emph{eBPF Options}
@gccoptlist{-mbig-endian -mlittle-endian -mkernel=@var{version}
-mframe-limit=@var{bytes} -mxbpf -mco-re -mno-co-re -mjmpext
--mjmp32 -malu32 -mcpu=@var{version} -masm=@var{dialect}}
+-mjmp32 -malu32 -mv3-atomics -mcpu=@var{version} -masm=@var{dialect}}
@emph{FR30 Options}
@gccoptlist{-msmall-model -mno-lsim}
@@ -24716,6 +24716,11 @@ Enable byte swap instructions. Enabled for CPU v4 and above.
Enable signed division and modulus instructions. Enabled for CPU v4
and above.
+@opindex mv3-atomics
+@item -mv3-atomics
+Enable instructions for general atomic operations introduced in CPU v3.
+Enabled for CPU v3 and above.
+
@opindex mcpu
@item -mcpu=@var{version}
This specifies which version of the eBPF ISA to target. Newer versions
@@ -24735,6 +24740,7 @@ All features of v2, plus:
@itemize @minus
@item 32-bit jump operations, as in @option{-mjmp32}
@item 32-bit ALU operations, as in @option{-malu32}
+@item general atomic operations, as in @option{-mv3-atomics}
@end itemize
@item v4
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-cmpxchg-1.c b/gcc/testsuite/gcc.target/bpf/atomic-cmpxchg-1.c
new file mode 100644
index 000000000000..4bb6a7dba294
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-cmpxchg-1.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-mv3-atomics -O2" } */
+
+int
+foo (int *p, int *expected, int desired)
+{
+ return __atomic_compare_exchange (p, expected, &desired, 0,
+ __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+}
+
+int
+foo64 (long *p, long *expected, long desired)
+{
+ return __atomic_compare_exchange (p, expected, &desired, 0,
+ __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "acmp\t.*" } } */
+/* { dg-final { scan-assembler "acmp32\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-cmpxchg-2.c b/gcc/testsuite/gcc.target/bpf/atomic-cmpxchg-2.c
new file mode 100644
index 000000000000..4036570ac601
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-cmpxchg-2.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-mno-v3-atomics -O2" } */
+
+int
+foo (int *p, int *expected, int desired)
+{
+ return __atomic_compare_exchange (p, expected, &desired, 0,
+ __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+}
+
+int
+foo64 (long *p, long *expected, long desired)
+{
+ return __atomic_compare_exchange (p, expected, &desired, 0,
+ __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler-not "acmp\t.*" } } */
+/* { dg-final { scan-assembler-not "acmp32\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-op-1.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-op-1.c
new file mode 100644
index 000000000000..533e955fe883
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-op-1.c
@@ -0,0 +1,50 @@
+/* Test 64-bit atomic-fetch-op instructions. */
+
+/* { dg-do compile } */
+/* { dg-options "-mv3-atomics -O2" } */
+
+long val;
+
+long
+test_atomic_fetch_add (long x)
+{
+ return __atomic_fetch_add (&val, x, __ATOMIC_ACQUIRE);
+}
+
+long
+test_atomic_fetch_sub (long x)
+{
+ return __atomic_fetch_sub (&val, x, __ATOMIC_RELEASE);
+}
+
+long
+test_atomic_fetch_and (long x)
+{
+ return __atomic_fetch_and (&val, x, __ATOMIC_ACQUIRE);
+}
+
+long
+test_atomic_fetch_nand (long x)
+{
+ return __atomic_fetch_nand (&val, x, __ATOMIC_ACQUIRE);
+}
+
+long
+test_atomic_fetch_or (long x)
+{
+ return __atomic_fetch_or (&val, x, __ATOMIC_ACQUIRE);
+}
+
+long
+test_atomic_fetch_xor (long x)
+{
+ return __atomic_fetch_xor (&val, x, __ATOMIC_ACQUIRE);
+}
+
+/* sub implemented in terms of add */
+/* { dg-final { scan-assembler-times "afadd\t" 2 } } */
+/* { dg-final { scan-assembler-times "afand\t" 1 } } */
+/* nand must use a compare-exchange loop */
+/* { dg-final { scan-assembler "acmp\t" } } */
+/* { dg-final { scan-assembler-times "afor\t" 1 } } */
+/* { dg-final { scan-assembler-times "afxor\t" 1 } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-op-2.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-op-2.c
new file mode 100644
index 000000000000..6b9ee6348b51
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-op-2.c
@@ -0,0 +1,50 @@
+/* Test 32-bit atomic-fetch-op instructions. */
+
+/* { dg-do compile } */
+/* { dg-options "-mv3-atomics -O2" } */
+
+int val;
+
+int
+test_atomic_fetch_add (int x)
+{
+ return __atomic_fetch_add (&val, x, __ATOMIC_ACQUIRE);
+}
+
+int
+test_atomic_fetch_sub (int x)
+{
+ return __atomic_fetch_sub (&val, x, __ATOMIC_RELEASE);
+}
+
+int
+test_atomic_fetch_and (int x)
+{
+ return __atomic_fetch_and (&val, x, __ATOMIC_ACQUIRE);
+}
+
+int
+test_atomic_fetch_nand (int x)
+{
+ return __atomic_fetch_nand (&val, x, __ATOMIC_ACQUIRE);
+}
+
+int
+test_atomic_fetch_or (int x)
+{
+ return __atomic_fetch_or (&val, x, __ATOMIC_ACQUIRE);
+}
+
+int
+test_atomic_fetch_xor (int x)
+{
+ return __atomic_fetch_xor (&val, x, __ATOMIC_ACQUIRE);
+}
+
+/* sub implemented in terms of add */
+/* { dg-final { scan-assembler-times "afadd32\t" 2 } } */
+/* { dg-final { scan-assembler-times "afand32\t" 1 } } */
+/* nand must use a compare-exchange loop */
+/* { dg-final { scan-assembler "acmp32\t" } } */
+/* { dg-final { scan-assembler-times "afor32\t" 1 } } */
+/* { dg-final { scan-assembler-times "afxor32\t" 1 } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-op-3.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-op-3.c
new file mode 100644
index 000000000000..044a2f76474b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-op-3.c
@@ -0,0 +1,49 @@
+/* Test atomic-fetch-op instructions are disabled with -mno-v3-atomics. */
+
+/* { dg-do compile } */
+/* { dg-options "-mno-v3-atomics -O2" } */
+
+long val;
+
+long
+test_atomic_fetch_add (long x)
+{
+ return __atomic_fetch_add (&val, x, __ATOMIC_ACQUIRE);
+}
+
+long
+test_atomic_fetch_sub (long x)
+{
+ return __atomic_fetch_sub (&val, x, __ATOMIC_RELEASE);
+}
+
+long
+test_atomic_fetch_and (long x)
+{
+ return __atomic_fetch_and (&val, x, __ATOMIC_ACQUIRE);
+}
+
+long
+test_atomic_fetch_nand (long x)
+{
+ return __atomic_fetch_nand (&val, x, __ATOMIC_ACQUIRE);
+}
+
+long
+test_atomic_fetch_or (long x)
+{
+ return __atomic_fetch_or (&val, x, __ATOMIC_ACQUIRE);
+}
+
+long
+test_atomic_fetch_xor (long x)
+{
+ return __atomic_fetch_xor (&val, x, __ATOMIC_ACQUIRE);
+}
+
+/* { dg-final { scan-assembler-not "afadd\t" } } */
+/* { dg-final { scan-assembler-not "afand\t" } } */
+/* { dg-final { scan-assembler-not "afor\t" } } */
+/* { dg-final { scan-assembler-not "afxor\t" } } */
+/* { dg-final { scan-assembler-not "acmp\t" } } */
+/* { dg-final { scan-assembler-not "axchg\t" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-op-1.c b/gcc/testsuite/gcc.target/bpf/atomic-op-1.c
new file mode 100644
index 000000000000..453c0ed47ce0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-op-1.c
@@ -0,0 +1,49 @@
+/* Test 64-bit non-fetch atomic operations. */
+/* { dg-do compile } */
+/* { dg-options "-mv3-atomics -O2" } */
+
+long val;
+
+void
+test_atomic_add (long x)
+{
+ __atomic_add_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+void
+test_atomic_sub (long x)
+{
+ __atomic_add_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+void
+test_atomic_and (long x)
+{
+ __atomic_and_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+void
+test_atomic_nand (long x)
+{
+ __atomic_nand_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+void
+test_atomic_or (long x)
+{
+ __atomic_or_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+void
+test_atomic_xor (long x)
+{
+ __atomic_xor_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+/* sub implemented in terms of add, and we output xadd to support older GAS. */
+/* { dg-final { scan-assembler-times "xadddw\t" 2 } } */
+/* { dg-final { scan-assembler-times "aand\t" 1 } } */
+/* nand must use an exchange loop */
+/* { dg-final { scan-assembler "acmp\t" } } */
+/* { dg-final { scan-assembler-times "aor\t" 1 } } */
+/* { dg-final { scan-assembler-times "axor\t" 1 } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-op-2.c b/gcc/testsuite/gcc.target/bpf/atomic-op-2.c
new file mode 100644
index 000000000000..daacf42c00bf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-op-2.c
@@ -0,0 +1,49 @@
+/* Test 32-bit non-fetch atomic operations. */
+/* { dg-do compile } */
+/* { dg-options "-mv3-atomics -O2" } */
+
+int val;
+
+void
+test_atomic_add (int x)
+{
+ __atomic_add_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+void
+test_atomic_sub (int x)
+{
+ __atomic_add_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+void
+test_atomic_and (int x)
+{
+ __atomic_and_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+void
+test_atomic_nand (int x)
+{
+ __atomic_nand_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+void
+test_atomic_or (int x)
+{
+ __atomic_or_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+void
+test_atomic_xor (int x)
+{
+ __atomic_xor_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+/* sub implemented in terms of add, and we output xadd to support older GAS. */
+/* { dg-final { scan-assembler-times "xaddw\t" 2 } } */
+/* { dg-final { scan-assembler-times "aand32\t" 1 } } */
+/* nand must use an exchange loop */
+/* { dg-final { scan-assembler "acmp32\t" } } */
+/* { dg-final { scan-assembler-times "aor32\t" 1 } } */
+/* { dg-final { scan-assembler-times "axor32\t" 1 } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-op-3.c b/gcc/testsuite/gcc.target/bpf/atomic-op-3.c
new file mode 100644
index 000000000000..b2ce28926347
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-op-3.c
@@ -0,0 +1,49 @@
+/* Test that atomic insns are properly disabled with -mno-v3-atomics. */
+/* { dg-do compile } */
+/* { dg-options "-mno-v3-atomics -O2" } */
+
+int val;
+
+void
+test_atomic_add (int x)
+{
+ __atomic_add_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+void
+test_atomic_sub (int x)
+{
+ __atomic_add_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+void
+test_atomic_and (int x)
+{
+ __atomic_and_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+void
+test_atomic_nand (int x)
+{
+ __atomic_nand_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+void
+test_atomic_or (int x)
+{
+ __atomic_or_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+void
+test_atomic_xor (int x)
+{
+ __atomic_xor_fetch (&val, x, __ATOMIC_ACQUIRE);
+}
+
+/* Without v3 atomics, only xadd{w,dw} is available. */
+/* { dg-final { scan-assembler-not "aadd" } } */
+/* { dg-final { scan-assembler-not "aand" } } */
+/* { dg-final { scan-assembler-not "aor" } } */
+/* { dg-final { scan-assembler-not "axor" } } */
+/* { dg-final { scan-assembler-not "axchg" } } */
+/* { dg-final { scan-assembler-not "acmp" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-xchg-1.c b/gcc/testsuite/gcc.target/bpf/atomic-xchg-1.c
new file mode 100644
index 000000000000..bab806393dfd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-xchg-1.c
@@ -0,0 +1,20 @@
+/* Test atomic exchange instruction. */
+/* { dg-do compile } */
+/* { dg-options "-mv3-atomics -O2" } */
+
+int foo (int *p, int *new)
+{
+ int old;
+ __atomic_exchange (p, new, &old, __ATOMIC_RELAXED);
+ return old;
+}
+
+int foo64 (long *p, long *new)
+{
+ long old;
+ __atomic_exchange (p, new, &old, __ATOMIC_SEQ_CST);
+ return old;
+}
+
+/* { dg-final { scan-assembler-times "axchg\t.*" 1 } } */
+/* { dg-final { scan-assembler-times "axchg32\t.*" 1 } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-xchg-2.c b/gcc/testsuite/gcc.target/bpf/atomic-xchg-2.c
new file mode 100644
index 000000000000..3b6324e966b8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-xchg-2.c
@@ -0,0 +1,20 @@
+/* Test atomic exchange instruction is disabled with -mno-v3-atomics. */
+/* { dg-do compile } */
+/* { dg-options "-mno-v3-atomics -O2" } */
+
+int foo (int *p, int *new)
+{
+ int old;
+ __atomic_exchange (p, new, &old, __ATOMIC_RELAXED);
+ return old;
+}
+
+int foo64 (long *p, long *new)
+{
+ long old;
+ __atomic_exchange (p, new, &old, __ATOMIC_SEQ_CST);
+ return old;
+}
+
+/* { dg-final { scan-assembler-not "axchg\t.*" } } */
+/* { dg-final { scan-assembler-not "axchg32\t.*" } } */
--
GitLab