diff --git a/.github/.cspell/project-dictionary.txt b/.github/.cspell/project-dictionary.txt index 1283bb11..90ac0d74 100644 --- a/.github/.cspell/project-dictionary.txt +++ b/.github/.cspell/project-dictionary.txt @@ -16,6 +16,7 @@ cdsg CDSY cinc clrex +cmpne cmpw cmpxchg cset @@ -41,6 +42,7 @@ ldar ldarx ldaxp ldclrp +ldex ldiapp ldrd ldrex @@ -107,6 +109,7 @@ srlv stbar stdarch stdcx +stex stilp stlxp stpq diff --git a/README.md b/README.md index 527072b2..a2406266 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ This crate provides a way to soundly perform such operations. ## Platform Support -Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, PowerPC, MSP430, AVR, SPARC, Hexagon, M68k, and Xtensa are supported. +Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, PowerPC, MSP430, AVR, SPARC, Hexagon, M68k, C-SKY, and Xtensa are supported. | target_arch | primitives | load/store | swap/CAS | | ------------------------------- | --------------------------------------------------- |:----------:|:--------:| @@ -44,6 +44,7 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, | sparc64 \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓ | | hexagon \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓ | | m68k \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓\[1] | +| csky \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓ | | xtensa \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓\[1] | \[1] Arm's atomic RMW operations are not available on Armv6-M (thumbv6m). RISC-V's atomic RMW operations are not available on targets without the A (or G which means IMAFD) extension such as riscv32i, riscv32imc, etc. M68k's atomic RMW operations requires target-cpu M68020+ (Linux is M68020 by default). Xtensa's atomic RMW operations are not available on esp32s2.
diff --git a/build.rs b/build.rs index cc4dc8f3..9d4d3f12 100644 --- a/build.rs +++ b/build.rs @@ -117,6 +117,15 @@ fn main() { println!("cargo:rustc-cfg=atomic_maybe_uninit_unstable_asm_experimental_arch"); } } + "csky" => { + // https://github.com/rust-lang/rust/pull/136217 merged in Rust 1.86 (nightly-2025-02-14). + if version.nightly + && version.probe(86, 2025, 2, 13) + && is_allowed_feature("asm_experimental_arch") + { + println!("cargo:rustc-cfg=atomic_maybe_uninit_unstable_asm_experimental_arch"); + } + } "hexagon" => { // https://github.com/rust-lang/rust/pull/133452 merged in Rust 1.85 (nightly-2024-11-29). if version.nightly diff --git a/src/arch/README.md b/src/arch/README.md index 7dc2aa08..433f18c6 100644 --- a/src/arch/README.md +++ b/src/arch/README.md @@ -8,6 +8,7 @@ This document describes the operations that are considered atomic by architectur - [AArch64](#aarch64) - [Arm](#arm) - [AVR](#avr) +- [C-SKY](#c-sky) - [Hexagon](#hexagon) - [LoongArch](#loongarch) - [M68k](#m68k) @@ -55,6 +56,13 @@ This architecture is always single-core and the following operations are atomic: disabling and restoring implementation must imply compiler fences, e.g., asm without nomem/readonly) may be moved out of the critical section by compiler optimizations. +## C-SKY + +target_arch: csky
+Implementation: [csky.rs](csky.rs)
+ +TODO + ## Hexagon target_arch: hexagon
diff --git a/src/arch/cfgs/csky.rs b/src/arch/cfgs/csky.rs new file mode 100644 index 00000000..bc666998 --- /dev/null +++ b/src/arch/cfgs/csky.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![allow(missing_docs)] + +#[macro_export] +macro_rules! cfg_has_atomic_8 { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_no_atomic_8 { + ($($tt:tt)*) => {}; +} +#[macro_export] +macro_rules! cfg_has_atomic_16 { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_no_atomic_16 { + ($($tt:tt)*) => {}; +} +#[macro_export] +macro_rules! cfg_has_atomic_32 { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_no_atomic_32 { + ($($tt:tt)*) => {}; +} +#[macro_export] +macro_rules! cfg_has_atomic_64 { + ($($tt:tt)*) => {}; +} +#[macro_export] +macro_rules! cfg_no_atomic_64 { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_has_atomic_128 { + ($($tt:tt)*) => {}; +} +#[macro_export] +macro_rules! cfg_no_atomic_128 { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_has_atomic_cas { + ($($tt:tt)*) => { $($tt)* }; +} +#[macro_export] +macro_rules! cfg_no_atomic_cas { + ($($tt:tt)*) => {}; +} diff --git a/src/arch/csky.rs b/src/arch/csky.rs new file mode 100644 index 00000000..90dcaf75 --- /dev/null +++ b/src/arch/csky.rs @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +/* +C-SKY + +Refs: +- CSKY Architecture user_guide + https://github.com/c-sky/csky-doc/blob/9f7121f7d40970ba5cc0f15716da033db2bb9d07/CSKY%20Architecture%20user_guide.pdf +- Linux kernel's C-SKY atomic implementation + https://github.com/torvalds/linux/blob/v6.11/arch/csky/include/asm/atomic.h + https://github.com/torvalds/linux/blob/v6.11/arch/csky/include/asm/barrier.h + https://github.com/torvalds/linux/blob/v6.11/arch/csky/include/asm/cmpxchg.h + +Generated asm: +- csky https://godbolt.org/z/jK4c68WeG +*/ + +#[path = "cfgs/csky.rs"] +mod cfgs; + +use core::{arch::asm, mem::MaybeUninit, sync::atomic::Ordering}; + +use crate::raw::{AtomicCompareExchange, AtomicLoad, AtomicStore, AtomicSwap}; + +macro_rules! atomic_rmw { + ($op:ident, $order:ident) => { + match $order { + Ordering::Relaxed => $op!("", ""), + Ordering::Acquire => $op!("sync32", ""), + Ordering::Release => $op!("", "sync32"), + Ordering::AcqRel | Ordering::SeqCst => $op!("sync32", "sync32"), + _ => unreachable!(), + } + }; +} + +#[rustfmt::skip] +macro_rules! atomic_load_store { + ($int_type:ident, $suffix:tt) => { + impl AtomicLoad for $int_type { + #[inline] + unsafe fn atomic_load( + src: *const MaybeUninit, + order: Ordering, + ) -> MaybeUninit { + let out: MaybeUninit; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + macro_rules! atomic_load { + ($acquire:tt, $release:tt) => { + asm!( + $release, // fence + concat!("ld32.", $suffix, " {out}, ({src}, 0x0)"), // atomic { out = *src } + $acquire, // fence + src = in(reg) ptr_reg!(src), + out = lateout(reg) out, + options(nostack, preserves_flags), + ) + }; + } + match order { + Ordering::Relaxed => atomic_load!("", ""), + Ordering::Acquire => atomic_load!("sync32", ""), + Ordering::SeqCst => atomic_load!("sync32", "sync32"), + _ => unreachable!(), + } + } + out + } + } + impl AtomicStore for $int_type { + #[inline] + unsafe fn atomic_store( + dst: *mut MaybeUninit, + val: MaybeUninit, + order: Ordering, + ) { + // SAFETY: the caller must uphold the safety contract. + unsafe { + macro_rules! store { + ($acquire:tt, $release:tt) => { + asm!( + $release, // fence + concat!("st32.", $suffix, " {val}, ({dst}, 0x0)"), // atomic { *dst = val } + $acquire, // fence + dst = in(reg) ptr_reg!(dst), + val = in(reg) val, + options(nostack, preserves_flags), + ) + }; + } + atomic_rmw!(store, order); + } + } + } + }; +} + +#[rustfmt::skip] +macro_rules! atomic { + ($int_type:ident) => { + atomic_load_store!($int_type, "w"); + impl AtomicSwap for $int_type { + #[inline] + unsafe fn atomic_swap( + dst: *mut MaybeUninit, + val: MaybeUninit, + order: Ordering, + ) -> MaybeUninit { + let mut out: MaybeUninit; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + macro_rules! swap { + ($acquire:tt, $release:tt) => { + asm!( + $release, // fence + "2:", // 'retry: + "ldex32.w {out}, ({dst}, 0)", // atomic { out = *dst; EXCLUSIVE = dst } + "or32 {tmp}, {val}, {val}", // tmp = val + "stex32.w {tmp}, ({dst}, 0)", // atomic { if EXCLUSIVE == dst { *dst = tmp; tmp = 1 } else { tmp = 0 }; EXCLUSIVE = None } + "bez32 {tmp}, 2b", // if tmp == 0 { jump 'retry } + $acquire, // fence + dst = in(reg) ptr_reg!(dst), + val = in(reg) val, + out = out(reg) out, + tmp = out(reg) _, + options(nostack, preserves_flags), + ) + }; + } + atomic_rmw!(swap, order); + } + out + } + } + impl AtomicCompareExchange for $int_type { + #[inline] + unsafe fn atomic_compare_exchange( + dst: *mut MaybeUninit, + old: MaybeUninit, + new: MaybeUninit, + success: Ordering, + failure: Ordering, + ) -> (MaybeUninit, bool) { + let order = crate::utils::upgrade_success_ordering(success, failure); + let mut out: MaybeUninit; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + let mut r: u32; + macro_rules! cmpxchg { + ($acquire:tt, $release:tt) => { + asm!( + $release, // fence + "2:", // 'retry: + "ldex32.w {out}, ({dst}, 0)", // atomic { out = *dst; EXCLUSIVE = dst } + "cmpne32 {out}, {old}", // if out != old { C = 1 } else { C = 0 } + "bt32 3f", // if C == 1 { jump 'cmp-fail } + "or32 {tmp}, {new}, {new}", // tmp = new + "stex32.w {tmp}, ({dst}, 0)", // atomic { if EXCLUSIVE == dst { *dst = tmp; tmp = 1 } else { tmp = 0 }; EXCLUSIVE = None } + "bez32 {tmp}, 2b", // if tmp == 0 { jump 'retry } + "br32 4f", // jump 'success + "3:", // 'cmp-fail: + "movi32 {tmp}, 0", // tmp = 0 + "4:", // 'success: + $acquire, // fence + dst = in(reg) ptr_reg!(dst), + old = in(reg) old, + new = in(reg) new, + out = out(reg) out, + tmp = out(reg) r, + // Do not use `preserves_flags` because CMPNE modifies condition bit C. + options(nostack), + ) + }; + } + atomic_rmw!(cmpxchg, order); + crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test + (out, r != 0) + } + } + } + }; +} + +#[rustfmt::skip] +macro_rules! atomic_sub_word { + ($int_type:ident, $suffix:tt) => { + atomic_load_store!($int_type, $suffix); + impl AtomicSwap for $int_type { + #[inline] + unsafe fn atomic_swap( + dst: *mut MaybeUninit, + val: MaybeUninit, + order: Ordering, + ) -> MaybeUninit { + let (dst, shift, mask) = crate::utils::create_sub_word_mask_values(dst); + let mut out: MaybeUninit; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + macro_rules! swap { + ($acquire:tt, $release:tt) => { + // Implement sub-word atomic operations using word-sized LL/SC loop. + // See also create_sub_word_mask_values. + asm!( + "lsl32 {val}, {val}, {shift}", // val <<= shift + $release, // fence + "2:", // 'retry: + "ldex32.w {out}, ({dst}, 0)", // atomic { out = *dst; EXCLUSIVE = dst } + "andn32 {tmp}, {out}, {mask}", // tmp = out & !mask + "or32 {tmp}, {tmp}, {val}", // tmp |= val + "stex32.w {tmp}, ({dst}, 0)", // atomic { if EXCLUSIVE == dst { *dst = tmp; tmp = 1 } else { tmp = 0 }; EXCLUSIVE = None } + "bez32 {tmp}, 2b", // if tmp == 0 { jump 'retry } + "lsr32 {out}, {out}, {shift}", // out >>= shift + $acquire, // fence + dst = in(reg) ptr_reg!(dst), + val = inout(reg) crate::utils::ZeroExtend::zero_extend(val) => _, + out = out(reg) out, + shift = in(reg) shift, + mask = in(reg) mask, + tmp = out(reg) _, + options(nostack, preserves_flags), + ) + }; + } + atomic_rmw!(swap, order); + } + out + } + } + impl AtomicCompareExchange for $int_type { + #[inline] + unsafe fn atomic_compare_exchange( + dst: *mut MaybeUninit, + old: MaybeUninit, + new: MaybeUninit, + success: Ordering, + failure: Ordering, + ) -> (MaybeUninit, bool) { + let order = crate::utils::upgrade_success_ordering(success, failure); + let (dst, shift, mask) = crate::utils::create_sub_word_mask_values(dst); + let mut out: MaybeUninit; + + // SAFETY: the caller must uphold the safety contract. + unsafe { + let mut r: u32; + macro_rules! cmpxchg { + ($acquire:tt, $release:tt) => { + // Implement sub-word atomic operations using word-sized LL/SC loop. + // See also create_sub_word_mask_values. + asm!( + "lsl32 {old}, {old}, {shift}", // old <<= shift + "lsl32 {new}, {new}, {shift}", // new <<= shift + $release, // fence + "2:", // 'retry: + "ldex32.w {tmp}, ({dst}, 0)", // atomic { tmp = *dst; EXCLUSIVE = dst } + "and32 {out}, {tmp}, {mask}", // out = tmp & mask + "cmpne32 {out}, {old}", // if out != old { C = 1 } else { C = 0 } + "bt32 3f", // if C == 1 { jump 'cmp-fail } + "andn32 {tmp}, {tmp}, {mask}", // tmp &= !mask + "or32 {tmp}, {tmp}, {new}", // tmp |= new + "stex32.w {tmp}, ({dst}, 0)", // atomic { if EXCLUSIVE == dst { *dst = tmp; tmp = 1 } else { tmp = 0 }; EXCLUSIVE = None } + "bez32 {tmp}, 2b", // if tmp == 0 { jump 'retry } + "br32 4f", // jump 'success + "3:", // 'cmp-fail: + "movi32 {tmp}, 0", // tmp = 0 + "4:", // 'success: + "lsr32 {out}, {out}, {shift}", // out >>= shift + $acquire, // fence + dst = in(reg) ptr_reg!(dst), + old = inout(reg) crate::utils::ZeroExtend::zero_extend(old) => _, + new = inout(reg) crate::utils::ZeroExtend::zero_extend(new) => _, + out = out(reg) out, + shift = in(reg) shift, + mask = in(reg) mask, + tmp = out(reg) r, + // Do not use `preserves_flags` because CMPNE modifies condition bit C. + options(nostack), + ) + }; + } + atomic_rmw!(cmpxchg, order); + crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test + (out, r != 0) + } + } + } + }; +} + +atomic_sub_word!(i8, "b"); +atomic_sub_word!(u8, "b"); +atomic_sub_word!(i16, "h"); +atomic_sub_word!(u16, "h"); +atomic!(i32); +atomic!(u32); +atomic!(isize); +atomic!(usize); diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 7ad4b261..522acf09 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -29,6 +29,7 @@ all( any( target_arch = "avr", + target_arch = "csky", target_arch = "hexagon", target_arch = "m68k", target_arch = "mips", @@ -86,6 +87,9 @@ mod armv8; #[cfg(target_arch = "avr")] #[cfg(atomic_maybe_uninit_unstable_asm_experimental_arch)] mod avr; +#[cfg(target_arch = "csky")] +#[cfg(atomic_maybe_uninit_unstable_asm_experimental_arch)] +mod csky; #[cfg(target_arch = "hexagon")] #[cfg(atomic_maybe_uninit_unstable_asm_experimental_arch)] mod hexagon; diff --git a/src/lib.rs b/src/lib.rs index 51249d79..d50646d3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,7 +15,7 @@ This crate provides a way to soundly perform such operations. ## Platform Support -Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, PowerPC, MSP430, AVR, SPARC, Hexagon, M68k, and Xtensa are supported. +Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, PowerPC, MSP430, AVR, SPARC, Hexagon, M68k, C-SKY, and Xtensa are supported. | target_arch | primitives | load/store | swap/CAS | | ------------------------------- | --------------------------------------------------- |:----------:|:--------:| @@ -41,6 +41,7 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch64, Arm64EC, s390x, MIPS, | sparc64 \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓ | | hexagon \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓ | | m68k \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓\[1] | +| csky \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓ | | xtensa \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓\[1] | \[1] Arm's atomic RMW operations are not available on Armv6-M (thumbv6m). RISC-V's atomic RMW operations are not available on targets without the A (or G which means IMAFD) extension such as riscv32i, riscv32imc, etc. M68k's atomic RMW operations requires target-cpu M68020+ (Linux is M68020 by default). Xtensa's atomic RMW operations are not available on esp32s2.
diff --git a/tools/build.sh b/tools/build.sh index 63c8bc2f..82cc43f0 100755 --- a/tools/build.sh +++ b/tools/build.sh @@ -68,6 +68,10 @@ default_targets=( # rustc -Z unstable-options --print all-target-specs-json | jq -r '. | to_entries[] | if .value.arch == "avr" then .key else empty end' avr-unknown-gnu-atmega2560 # custom target + # csky + # rustc -Z unstable-options --print all-target-specs-json | jq -r '. | to_entries[] | if .value.arch == "csky" then .key else empty end' + csky-unknown-linux-gnuabiv2 + # hexagon # rustc -Z unstable-options --print all-target-specs-json | jq -r '. | to_entries[] | if .value.arch == "hexagon" then .key else empty end' hexagon-unknown-linux-musl