riscv: Provide all operation when Zalrsc extension is available even …

…if A extension is not available
taiki-e · Feb 23, 2025 · 7472093 · 7472093
1 parent 0de7e99
commit 7472093
Show file tree

Hide file tree

Showing 7 changed files with 123 additions and 23 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -575,6 +575,12 @@ jobs:
           RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+zabha,+zacas
           QEMU_CPU: max
         if: startsWith(matrix.target, 'riscv')
+      # riscv zalrsc-based RMW ({32,64}-bit swap with Zalrsc without Zaamo)
+      - run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
+        env:
+          RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} --cfg atomic_maybe_uninit_test_prefer_zalrsc_over_zaamo
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} --cfg atomic_maybe_uninit_test_prefer_zalrsc_over_zaamo
+        if: startsWith(matrix.target, 'riscv')
       # s390x z196 (arch9)
       - run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
         env:

diff --git a/Cargo.toml b/Cargo.toml
@@ -66,7 +66,7 @@ unexpected_cfgs = { level = "warn", check-cfg = [
     'cfg(target_pointer_width,values("128"))',
     # Known custom cfgs, excluding those that may be set by build script.
     # Not public API.
-    'cfg(qemu,valgrind)',
+    'cfg(atomic_maybe_uninit_test_prefer_zalrsc_over_zaamo,qemu,valgrind)',
     # Public APIs, considered unstable unless documented in readme.
     # arm: Use cp15_barrier instead of __kuser_memory_barrier on Armv6 Linux/Android.
     # Armv6 binaries compiled with this cfg may cause problems when run on Armv7+ chips:

diff --git a/README.md b/README.md
@@ -46,7 +46,7 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS,
 | m68k \[8] (experimental)        | isize,usize,i8,u8,i16,u16,i32,u32                   | ✓          | ✓\[1]    |
 | xtensa \[8] (experimental)      | isize,usize,i8,u8,i16,u16,i32,u32                   | ✓          | ✓\[1]    |
 
-\[1] Arm's atomic RMW operations are not available on Armv6-M (thumbv6m). RISC-V's atomic RMW operations are not available on targets without the A (or G which means IMAFD) extension such as riscv32i, riscv32imc, etc. M68k's atomic RMW operations requires target-cpu M68020+ (Linux is M68020 by default). Xtensa's atomic RMW operations are not available on esp32s2.<br>
+\[1] Arm's atomic RMW operations are not available on Armv6-M (thumbv6m). RISC-V's atomic RMW operations are not available on targets without the A (or G which means IMAFD) or Zalrsc extension, such as riscv32i, riscv32imc, etc. M68k's atomic RMW operations requires target-cpu M68020+ (Linux is M68020 by default). Xtensa's atomic RMW operations are not available on esp32s2.<br>
 \[2] Requires `cmpxchg16b` target feature (enabled by default on Apple and Windows (except Windows 7) targets).<br>
 \[3] Armv6+ or Linux/Android, except for M-profile architecture such as thumbv6m, thumbv7m, etc.<br>
 \[4] Requires `quadword-atomics` target feature (enabled by default on powerpc64le).<br>

diff --git a/src/arch/cfgs/riscv.rs b/src/arch/cfgs/riscv.rs
@@ -54,22 +54,42 @@ macro_rules! cfg_has_atomic_128 {
 macro_rules! cfg_no_atomic_128 {
     ($($tt:tt)*) => { $($tt)* };
 }
-#[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
+#[cfg(any(
+    target_feature = "a",
+    atomic_maybe_uninit_target_feature = "a",
+    target_feature = "zalrsc",
+    atomic_maybe_uninit_target_feature = "zalrsc",
+))]
 #[macro_export]
 macro_rules! cfg_has_atomic_cas {
     ($($tt:tt)*) => { $($tt)* };
 }
-#[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
+#[cfg(any(
+    target_feature = "a",
+    atomic_maybe_uninit_target_feature = "a",
+    target_feature = "zalrsc",
+    atomic_maybe_uninit_target_feature = "zalrsc",
+))]
 #[macro_export]
 macro_rules! cfg_no_atomic_cas {
     ($($tt:tt)*) => {};
 }
-#[cfg(not(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a")))]
+#[cfg(not(any(
+    target_feature = "a",
+    atomic_maybe_uninit_target_feature = "a",
+    target_feature = "zalrsc",
+    atomic_maybe_uninit_target_feature = "zalrsc",
+)))]
 #[macro_export]
 macro_rules! cfg_has_atomic_cas {
     ($($tt:tt)*) => {};
 }
-#[cfg(not(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a")))]
+#[cfg(not(any(
+    target_feature = "a",
+    atomic_maybe_uninit_target_feature = "a",
+    target_feature = "zalrsc",
+    atomic_maybe_uninit_target_feature = "zalrsc",
+)))]
 #[macro_export]
 macro_rules! cfg_no_atomic_cas {
     ($($tt:tt)*) => { $($tt)* };

diff --git a/src/arch/riscv.rs b/src/arch/riscv.rs
@@ -24,7 +24,6 @@ Generated asm:
 // TODO:
 // - 64-bit/128-bit atomics on RV32/RV64 with Zacas
 // - {8,16}-bit swap/cas with Zacas without Zalrsc & Zabha (use amocas.w)
-// - {32,64}-bit swap with Zalrsc without Zaamo (use lr/sc)
 
 #[path = "cfgs/riscv.rs"]
 mod cfgs;
@@ -56,6 +55,7 @@ use crate::raw::AtomicSwap;
 use crate::raw::{AtomicLoad, AtomicStore};
 
 #[cfg(not(all(
+    not(atomic_maybe_uninit_test_prefer_zalrsc_over_zaamo),
     any(target_feature = "zabha", atomic_maybe_uninit_target_feature = "zabha"),
     any(target_feature = "zacas", atomic_maybe_uninit_target_feature = "zacas"),
 )))]
@@ -72,6 +72,7 @@ macro_rules! w {
     };
 }
 #[cfg(not(all(
+    not(atomic_maybe_uninit_test_prefer_zalrsc_over_zaamo),
     any(target_feature = "zabha", atomic_maybe_uninit_target_feature = "zabha"),
     any(target_feature = "zacas", atomic_maybe_uninit_target_feature = "zacas"),
 )))]
@@ -88,11 +89,14 @@ macro_rules! w {
     };
 }
 
-#[cfg(any(
-    target_feature = "a",
-    atomic_maybe_uninit_target_feature = "a",
-    target_feature = "zaamo",
-    atomic_maybe_uninit_target_feature = "zaamo",
+#[cfg(all(
+    not(atomic_maybe_uninit_test_prefer_zalrsc_over_zaamo),
+    any(
+        target_feature = "a",
+        atomic_maybe_uninit_target_feature = "a",
+        target_feature = "zaamo",
+        atomic_maybe_uninit_target_feature = "zaamo",
+    ),
 ))]
 macro_rules! atomic_rmw_amo {
     ($op:ident, $order:ident) => {
@@ -124,6 +128,7 @@ macro_rules! atomic_rmw_amocas {
     };
 }
 #[cfg(not(all(
+    not(atomic_maybe_uninit_test_prefer_zalrsc_over_zaamo),
     any(target_feature = "zabha", atomic_maybe_uninit_target_feature = "zabha"),
     any(target_feature = "zacas", atomic_maybe_uninit_target_feature = "zacas"),
 )))]
@@ -218,11 +223,14 @@ macro_rules! atomic_load_store {
     };
 }
 
-#[cfg(any(
-    target_feature = "a",
-    atomic_maybe_uninit_target_feature = "a",
-    target_feature = "zaamo",
-    atomic_maybe_uninit_target_feature = "zaamo",
+#[cfg(all(
+    not(atomic_maybe_uninit_test_prefer_zalrsc_over_zaamo),
+    any(
+        target_feature = "a",
+        atomic_maybe_uninit_target_feature = "a",
+        target_feature = "zaamo",
+        atomic_maybe_uninit_target_feature = "zaamo",
+    ),
 ))]
 #[rustfmt::skip]
 macro_rules! atomic_zaamo {
@@ -311,13 +319,64 @@ macro_rules! atomic {
         atomic_load_store!($ty, $size);
 
         // swap
+        #[cfg(all(
+            not(atomic_maybe_uninit_test_prefer_zalrsc_over_zaamo),
+            any(
+                target_feature = "a",
+                atomic_maybe_uninit_target_feature = "a",
+                target_feature = "zaamo",
+                atomic_maybe_uninit_target_feature = "zaamo",
+            ),
+        ))]
+        atomic_zaamo!($ty, $size);
+        #[cfg(not(all(
+            not(atomic_maybe_uninit_test_prefer_zalrsc_over_zaamo),
+            any(
+                target_feature = "a",
+                atomic_maybe_uninit_target_feature = "a",
+                target_feature = "zaamo",
+                atomic_maybe_uninit_target_feature = "zaamo",
+            ),
+        )))]
         #[cfg(any(
             target_feature = "a",
             atomic_maybe_uninit_target_feature = "a",
-            target_feature = "zaamo",
-            atomic_maybe_uninit_target_feature = "zaamo",
+            target_feature = "zalrsc",
+            atomic_maybe_uninit_target_feature = "zalrsc",
         ))]
-        atomic_zaamo!($ty, $size);
+        impl AtomicSwap for $ty {
+            #[inline]
+            unsafe fn atomic_swap(
+                dst: *mut MaybeUninit<Self>,
+                val: MaybeUninit<Self>,
+                order: Ordering,
+            ) -> MaybeUninit<Self> {
+                debug_assert!(dst as usize % mem::size_of::<$ty>() == 0);
+                let mut out: MaybeUninit<Self>;
+
+                // SAFETY: the caller must uphold the safety contract,
+                // the cfg guarantee that the CPU supports Zalrsc extension.
+                unsafe {
+                    macro_rules! swap {
+                        ($acquire:tt, $release:tt) => {
+                            asm!(
+                                "2:", // 'retry:
+                                    concat!("lr.", $size, $acquire, " {out}, 0({dst})"),      // atomic { out = *dst; RS = dst }
+                                    concat!("sc.", $size, $release, " {r}, {val}, 0({dst})"), // atomic { if RS == dst { *dst = val; r = 0 } else { r = nonzero }; RS = None }
+                                    "bnez {r}, 2b",                                           // if r != 0 { jump 'retry }
+                                dst = in(reg) ptr_reg!(dst),
+                                val = in(reg) val,
+                                out = out(reg) out,
+                                r = out(reg) _,
+                                options(nostack, preserves_flags),
+                            )
+                        };
+                    }
+                    atomic_rmw_lr_sc!(swap, order);
+                }
+                out
+            }
+        }
 
         // compare_exchange
         #[cfg(any(target_feature = "zacas", atomic_maybe_uninit_target_feature = "zacas"))]
@@ -381,9 +440,15 @@ macro_rules! atomic_sub_word {
         atomic_load_store!($ty, $size);
 
         // swap
-        #[cfg(any(target_feature = "zabha", atomic_maybe_uninit_target_feature = "zabha"))]
+        #[cfg(all(
+            not(atomic_maybe_uninit_test_prefer_zalrsc_over_zaamo),
+            any(target_feature = "zabha", atomic_maybe_uninit_target_feature = "zabha"),
+        ))]
         atomic_zaamo!($ty, $size);
-        #[cfg(not(any(target_feature = "zabha", atomic_maybe_uninit_target_feature = "zabha")))]
+        #[cfg(not(all(
+            not(atomic_maybe_uninit_test_prefer_zalrsc_over_zaamo),
+            any(target_feature = "zabha", atomic_maybe_uninit_target_feature = "zabha"),
+        )))]
         #[cfg(any(
             target_feature = "a",
             atomic_maybe_uninit_target_feature = "a",

diff --git a/src/lib.rs b/src/lib.rs
@@ -43,7 +43,7 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS,
 | m68k \[8] (experimental)        | isize,usize,i8,u8,i16,u16,i32,u32                   | ✓          | ✓\[1]    |
 | xtensa \[8] (experimental)      | isize,usize,i8,u8,i16,u16,i32,u32                   | ✓          | ✓\[1]    |
 
-\[1] Arm's atomic RMW operations are not available on Armv6-M (thumbv6m). RISC-V's atomic RMW operations are not available on targets without the A (or G which means IMAFD) extension such as riscv32i, riscv32imc, etc. M68k's atomic RMW operations requires target-cpu M68020+ (Linux is M68020 by default). Xtensa's atomic RMW operations are not available on esp32s2.<br>
+\[1] Arm's atomic RMW operations are not available on Armv6-M (thumbv6m). RISC-V's atomic RMW operations are not available on targets without the A (or G which means IMAFD) or Zalrsc extension, such as riscv32i, riscv32imc, etc. M68k's atomic RMW operations requires target-cpu M68020+ (Linux is M68020 by default). Xtensa's atomic RMW operations are not available on esp32s2.<br>
 \[2] Requires `cmpxchg16b` target feature (enabled by default on Apple and Windows (except Windows 7) targets).<br>
 \[3] Armv6+ or Linux/Android, except for M-profile architecture such as thumbv6m, thumbv7m, etc.<br>
 \[4] Requires `quadword-atomics` target feature (enabled by default on powerpc64le).<br>

diff --git a/tools/no-std.sh b/tools/no-std.sh
@@ -265,6 +265,15 @@ run() {
       RUSTFLAGS="${target_rustflags}" \
       x_cargo "${args[@]}" --release "$@"
     case "${target}" in
+      riscv??[ie]-* | riscv??[ie]m-* | riscv??[ie]mc-*)
+        # With Zalrsc without Zaamo
+        CARGO_TARGET_DIR="${target_dir}/no-std-test-zalrsc" \
+          RUSTFLAGS="${target_rustflags} -C target-feature=+zalrsc" \
+          x_cargo "${args[@]}" "$@"
+        CARGO_TARGET_DIR="${target_dir}/no-std-test-zalrsc" \
+          RUSTFLAGS="${target_rustflags} -C target-feature=+zalrsc" \
+          x_cargo "${args[@]}" --release "$@"
+        ;;
       avr*)
         # Run with qemu-system-avr.
         subcmd=run