Skip to content

Commit

Permalink
riscv: Support Zacas and Zalrsc extensions
Browse files Browse the repository at this point in the history
  • Loading branch information
@taiki-e
taiki-e committed Feb 23, 2025
1 parent 3bdf4e0 commit 2a0901e
Show file tree
Hide file tree
Showing 6 changed files with 216 additions and 42 deletions.
1 change: 1 addition & 0 deletions .github/.cspell/project-dictionary.txt
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
addi
amocas
amoswap
amswap
andc
Expand Down
7 changes: 7 additions & 0 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -568,6 +568,13 @@ jobs:
RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+zabha
QEMU_CPU: max
if: startsWith(matrix.target, 'riscv')
# riscv +zabha,+zacas
- run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
env:
RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+zabha,+zacas
RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+zabha,+zacas
QEMU_CPU: max
if: startsWith(matrix.target, 'riscv')
# s390x z196 (arch9)
- run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
env:
Expand Down
2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ unexpected_cfgs = { level = "warn", check-cfg = [
'cfg(target_arch,values("amdgpu"))', # 1.86+ https://github.com/rust-lang/rust/pull/134740
'cfg(target_feature,values("lse2","lse128","rcpc3"))', # 1.82+ https://github.com/rust-lang/rust/pull/128192
'cfg(target_feature,values("partword-atomics","quadword-atomics"))', # 1.83+ https://github.com/rust-lang/rust/pull/130873
'cfg(target_feature,values("zaamo","zabha"))', # 1.83+ https://github.com/rust-lang/rust/pull/130877
'cfg(target_feature,values("zaamo","zabha","zalrsc"))', # 1.83+ https://github.com/rust-lang/rust/pull/130877
'cfg(target_feature,values("leoncasa","v9"))', # 1.84+ https://github.com/rust-lang/rust/pull/132552
'cfg(target_feature,values("x87"))', # 1.85+ https://github.com/rust-lang/rust/pull/133099
'cfg(target_feature,values("isa-68020"))', # 1.85+ https://github.com/rust-lang/rust/pull/134329
Expand Down
19 changes: 16 additions & 3 deletions build.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@ fn main() {
};

if version.minor >= 80 {
println!(r#"cargo:rustc-check-cfg=cfg(target_feature,values("v8m","fast-serialization"))"#);
println!(
r#"cargo:rustc-check-cfg=cfg(target_feature,values("v8m","fast-serialization","zacas"))"#
);

// Custom cfgs set by build script. Not public API.
// grep -F 'cargo:rustc-cfg=' build.rs | grep -Ev '^ *//' | sed -E 's/^.*cargo:rustc-cfg=//; s/(=\\)?".*$//' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/'
Expand All @@ -39,7 +41,7 @@ fn main() {
// TODO: handle multi-line target_feature_fallback
// grep -F 'target_feature_fallback("' build.rs | grep -Ev '^ *//' | sed -E 's/^.*target_feature_fallback\(//; s/",.*$/"/' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/'
println!(
r#"cargo:rustc-check-cfg=cfg(atomic_maybe_uninit_target_feature,values("a","cmpxchg16b","fast-serialization","isa-68020","leoncasa","lse","lse128","lse2","mclass","partword-atomics","quadword-atomics","rcpc","rcpc3","v5te","v6","v7","v8","v8m","v9","x87","zaamo","zabha"))"#
r#"cargo:rustc-check-cfg=cfg(atomic_maybe_uninit_target_feature,values("a","cmpxchg16b","fast-serialization","isa-68020","leoncasa","lse","lse128","lse2","mclass","partword-atomics","quadword-atomics","rcpc","rcpc3","v5te","v6","v7","v8","v8m","v9","x87","zaamo","zabha","zacas","zalrsc"))"#
);
}

Expand Down Expand Up @@ -300,7 +302,15 @@ fn main() {
// https://github.com/gcc-mirror/gcc/blob/08693e29ec186fd7941d0b73d4d466388971fe2f/gcc/config/riscv/arch-canonicalize#L45-L46
// https://github.com/rust-lang/rust/pull/130877
let mut zaamo = false;
// target_feature "zaamo"/"zabha" is unstable and available on rustc side since nightly-2024-10-02: https://github.com/rust-lang/rust/pull/130877
// As of rustc 1.84, target_feature "zacas" is not available on rustc side:
// https://github.com/rust-lang/rust/blob/1.84.0/compiler/rustc_target/src/target_features.rs#L425
if version.llvm >= 20 {
// amocas.{w,d,q} (and amocas.{b,h} if zabha is also available)
// available as experimental since LLVM 17 https://github.com/llvm/llvm-project/commit/29f630a1ddcbb03caa31b5002f0cbc105ff3a869
// available non-experimental since LLVM 20 https://github.com/llvm/llvm-project/commit/614aeda93b2225c6eb42b00ba189ba7ca2585c60
zaamo |= target_feature_fallback("zacas", false);
}
// target_feature "zaamo"/"zabha"/"zalrsc" is unstable and available on rustc side since nightly-2024-10-02: https://github.com/rust-lang/rust/pull/130877
if (!version.probe(83, 2024, 10, 1) || needs_target_feature_fallback(&version, None))
&& version.llvm >= 19
{
Expand All @@ -310,6 +320,9 @@ fn main() {
// amo*.{w,d}
// available since LLVM 19 https://github.com/llvm/llvm-project/commit/1a14c446dd800b1d79fed1735c48e392d06e495d / https://github.com/llvm/llvm-project/commit/8be079cdddfd628d356d9ddb5ab397ea95fb1030
target_feature_fallback("zaamo", zaamo);
// {lr,sc}.{w,d}
// available since LLVM 19 https://github.com/llvm/llvm-project/commit/1a14c446dd800b1d79fed1735c48e392d06e495d / https://github.com/llvm/llvm-project/commit/8be079cdddfd628d356d9ddb5ab397ea95fb1030
target_feature_fallback("zalrsc", false);
}
// Ratified RISC-V target features stabilized in Rust 1.75. https://github.com/rust-lang/rust/pull/116485
if needs_target_feature_fallback(&version, Some(75)) {
Expand Down
179 changes: 157 additions & 22 deletions src/arch/riscv.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,16 @@ Refs:
https://github.com/taiki-e/portable-atomic
Generated asm:
- riscv64gc https://godbolt.org/z/nfjcYcn1a
- riscv64gc (+zabha) https://godbolt.org/z/9En1f3G8v
- riscv32imac https://godbolt.org/z/3v8Yq7Ejh
- riscv32imac (+zabha) https://godbolt.org/z/9xWrW4Ynn
- riscv64gc https://godbolt.org/z/bqzMavsnh
- riscv64gc (+zabha) https://godbolt.org/z/KccGEjrdr
- riscv64gc (+zabha,+zacas) https://godbolt.org/z/7G8hjfoaz
- riscv32imac https://godbolt.org/z/sjWWhE7r7
*/

// TODO: Zacas extension, and Zalrsc extension without A extension
// TODO:
// - 64-bit/128-bit atomics on RV32/RV64 with Zacas
// - {8,16}-bit swap/cas with Zacas without Zalrsc & Zabha (use amocas.w)
// - {32,64}-bit swap with Zalrsc without Zaamo (use lr/sc)

#[path = "cfgs/riscv.rs"]
mod cfgs;
Expand All @@ -32,25 +35,44 @@ use core::{
sync::atomic::Ordering,
};

#[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
#[cfg(any(
target_feature = "a",
atomic_maybe_uninit_target_feature = "a",
target_feature = "zalrsc",
atomic_maybe_uninit_target_feature = "zalrsc",
target_feature = "zacas",
atomic_maybe_uninit_target_feature = "zacas",
))]
use crate::raw::AtomicCompareExchange;
#[cfg(any(
target_feature = "a",
atomic_maybe_uninit_target_feature = "a",
target_feature = "zaamo",
atomic_maybe_uninit_target_feature = "zaamo",
target_feature = "zalrsc",
atomic_maybe_uninit_target_feature = "zalrsc",
))]
use crate::raw::AtomicSwap;
use crate::raw::{AtomicLoad, AtomicStore};

#[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
#[cfg(any(
target_feature = "a",
atomic_maybe_uninit_target_feature = "a",
target_feature = "zalrsc",
atomic_maybe_uninit_target_feature = "zalrsc",
))]
#[cfg(target_arch = "riscv32")]
macro_rules! w {
() => {
""
};
}
#[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
#[cfg(any(
target_feature = "a",
atomic_maybe_uninit_target_feature = "a",
target_feature = "zalrsc",
atomic_maybe_uninit_target_feature = "zalrsc",
))]
#[cfg(target_arch = "riscv64")]
macro_rules! w {
() => {
Expand All @@ -76,7 +98,29 @@ macro_rules! atomic_rmw_amo {
}
};
}
#[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
#[cfg(any(target_feature = "zacas", atomic_maybe_uninit_target_feature = "zacas"))]
macro_rules! atomic_rmw_amocas {
($op:ident, $order:ident) => {
atomic_rmw_amocas!($op, $order, failure = $order)
};
($op:ident, $order:ident, failure = $failure:ident) => {
match $order {
Ordering::Relaxed => $op!("", ""),
Ordering::Acquire => $op!("", ".aq"),
Ordering::Release => $op!("", ".rl"),
Ordering::AcqRel => $op!("", ".aqrl"),
Ordering::SeqCst if $failure == Ordering::SeqCst => $op!("fence rw,rw", ".aqrl"),
Ordering::SeqCst => $op!("", ".aqrl"),
_ => unreachable!(),
}
};
}
#[cfg(any(
target_feature = "a",
atomic_maybe_uninit_target_feature = "a",
target_feature = "zalrsc",
atomic_maybe_uninit_target_feature = "zalrsc",
))]
macro_rules! atomic_rmw_lr_sc {
($op:ident, $order:ident) => {
match $order {
Expand Down Expand Up @@ -162,15 +206,15 @@ macro_rules! atomic_load_store {
};
}

#[cfg(any(
target_feature = "a",
atomic_maybe_uninit_target_feature = "a",
target_feature = "zaamo",
atomic_maybe_uninit_target_feature = "zaamo",
))]
#[rustfmt::skip]
macro_rules! atomic_swap {
macro_rules! atomic_zaamo {
($ty:ident, $size:tt) => {
#[cfg(any(
target_feature = "a",
atomic_maybe_uninit_target_feature = "a",
target_feature = "zaamo",
atomic_maybe_uninit_target_feature = "zaamo",
))]
impl AtomicSwap for $ty {
#[inline]
unsafe fn atomic_swap(
Expand All @@ -181,7 +225,8 @@ macro_rules! atomic_swap {
debug_assert!(dst as usize % mem::size_of::<$ty>() == 0);
let out: MaybeUninit<Self>;

// SAFETY: the caller must uphold the safety contract.
// SAFETY: the caller must uphold the safety contract,
// the cfg guarantee that the CPU supports Zaamo extension.
unsafe {
macro_rules! swap {
($order:tt) => {
Expand All @@ -201,13 +246,80 @@ macro_rules! atomic_swap {
}
};
}
#[cfg(any(
target_feature = "zacas",
atomic_maybe_uninit_target_feature = "zacas",
))]
#[rustfmt::skip]
macro_rules! atomic_zacas {
($ty:ident, $size:tt) => {
impl AtomicCompareExchange for $ty {
#[inline]
unsafe fn atomic_compare_exchange(
dst: *mut MaybeUninit<Self>,
old: MaybeUninit<Self>,
new: MaybeUninit<Self>,
success: Ordering,
failure: Ordering,
) -> (MaybeUninit<Self>, bool) {
debug_assert!(dst as usize % mem::size_of::<$ty>() == 0);
let order = crate::utils::upgrade_success_ordering(success, failure);
let mut out: MaybeUninit<Self>;

// SAFETY: the caller must uphold the safety contract,
// the cfg guarantee that the CPU supports Zacas extension.
unsafe {
let mut r: crate::utils::RegSize;
macro_rules! cmpxchg {
($fence:tt, $asm_order:tt) => {
asm!(
$fence, // fence
"mv {out}, {old}", // out = old
concat!("amocas.", $size, $asm_order, " {out}, {new}, 0({dst})"), // atomic { if *dst == old { *dst = new } else { old = *dst } }
"xor {r}, {out}, {old}", // r = out ^ old
"seqz {r}, {r}", // if r == 0 { r = 1 } else { r = 0 }
dst = in(reg) ptr_reg!(dst),
old = in(reg) old,
new = in(reg) new,
out = out(reg) out,
r = lateout(reg) r,
options(nostack, preserves_flags),
)
};
}
atomic_rmw_amocas!(cmpxchg, order, failure = failure);
crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
(out, r != 0)
}
}
}
};
}

#[rustfmt::skip]
macro_rules! atomic {
($ty:ident, $size:tt) => {
atomic_load_store!($ty, $size);
atomic_swap!($ty, $size);
#[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]

// swap
#[cfg(any(
target_feature = "a",
atomic_maybe_uninit_target_feature = "a",
target_feature = "zaamo",
atomic_maybe_uninit_target_feature = "zaamo",
))]
atomic_zaamo!($ty, $size);

// compare_exchange
#[cfg(any(target_feature = "zacas", atomic_maybe_uninit_target_feature = "zacas"))]
atomic_zacas!($ty, $size);
#[cfg(not(any(target_feature = "zacas", atomic_maybe_uninit_target_feature = "zacas")))]
#[cfg(any(
target_feature = "a",
atomic_maybe_uninit_target_feature = "a",
target_feature = "zalrsc",
atomic_maybe_uninit_target_feature = "zalrsc",
))]
impl AtomicCompareExchange for $ty {
#[inline]
unsafe fn atomic_compare_exchange(
Expand Down Expand Up @@ -257,10 +369,17 @@ macro_rules! atomic {
macro_rules! atomic_sub_word {
($ty:ident, $size:tt) => {
atomic_load_store!($ty, $size);

// swap
#[cfg(any(target_feature = "zabha", atomic_maybe_uninit_target_feature = "zabha"))]
atomic_swap!($ty, $size);
atomic_zaamo!($ty, $size);
#[cfg(not(any(target_feature = "zabha", atomic_maybe_uninit_target_feature = "zabha")))]
#[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
#[cfg(any(
target_feature = "a",
atomic_maybe_uninit_target_feature = "a",
target_feature = "zalrsc",
atomic_maybe_uninit_target_feature = "zalrsc",
))]
impl AtomicSwap for $ty {
#[inline]
unsafe fn atomic_swap(
Expand Down Expand Up @@ -305,7 +424,23 @@ macro_rules! atomic_sub_word {
out
}
}
#[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]

// compare_exchange
#[cfg(all(
any(target_feature = "zabha", atomic_maybe_uninit_target_feature = "zabha"),
any(target_feature = "zacas", atomic_maybe_uninit_target_feature = "zacas"),
))]
atomic_zacas!($ty, $size);
#[cfg(not(all(
any(target_feature = "zabha", atomic_maybe_uninit_target_feature = "zabha"),
any(target_feature = "zacas", atomic_maybe_uninit_target_feature = "zacas"),
)))]
#[cfg(any(
target_feature = "a",
atomic_maybe_uninit_target_feature = "a",
target_feature = "zalrsc",
atomic_maybe_uninit_target_feature = "zalrsc",
))]
impl AtomicCompareExchange for $ty {
#[inline]
unsafe fn atomic_compare_exchange(
Expand Down
Loading

0 comments on commit 2a0901e

Please sign in to comment.