diff --git a/libc/bionic/heap_tagging.cpp b/libc/bionic/heap_tagging.cpp index 0eaf34e00..62b5f5cbf 100644 --- a/libc/bionic/heap_tagging.cpp +++ b/libc/bionic/heap_tagging.cpp @@ -33,11 +33,11 @@ #include #include +extern "C" void scudo_malloc_disable_memory_tagging(); + static HeapTaggingLevel heap_tagging_level = M_HEAP_TAGGING_LEVEL_NONE; void SetDefaultHeapTaggingLevel() { - // Allow the kernel to accept tagged pointers in syscall arguments. This is a no-op (kernel - // returns -EINVAL) if the kernel doesn't understand the prctl. #if defined(__aarch64__) #define PR_SET_TAGGED_ADDR_CTRL 55 #define PR_TAGGED_ADDR_ENABLE (1UL << 0) @@ -47,15 +47,23 @@ void SetDefaultHeapTaggingLevel() { // syscall arguments. if (prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_ASYNC | (1 << PR_MTE_EXCL_SHIFT), 0, 0, 0) == 0) { + heap_tagging_level = M_HEAP_TAGGING_LEVEL_ASYNC; return; } #endif // ANDROID_EXPERIMENTAL_MTE + // Allow the kernel to accept tagged pointers in syscall arguments. This is a no-op (kernel + // returns -EINVAL) if the kernel doesn't understand the prctl. if (prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, 0, 0, 0) == 0) { +#if !__has_feature(hwaddress_sanitizer) heap_tagging_level = M_HEAP_TAGGING_LEVEL_TBI; __libc_globals.mutate([](libc_globals* globals) { - globals->heap_pointer_tag = reinterpret_cast(POINTER_TAG) << TAG_SHIFT; + // Arrange for us to set pointer tags to POINTER_TAG, check tags on + // deallocation and untag when passing pointers to the allocator. + globals->heap_pointer_tag = (reinterpret_cast(POINTER_TAG) << TAG_SHIFT) | + (0xffull << CHECK_SHIFT) | (0xffull << UNTAG_SHIFT); }); +#endif // hwaddress_sanitizer } #endif // aarch64 } @@ -66,16 +74,22 @@ bool SetHeapTaggingLevel(void* arg, size_t arg_size) { } auto tag_level = *reinterpret_cast(arg); + if (tag_level == heap_tagging_level) { + return true; + } + switch (tag_level) { case M_HEAP_TAGGING_LEVEL_NONE: break; case M_HEAP_TAGGING_LEVEL_TBI: + case M_HEAP_TAGGING_LEVEL_ASYNC: if (heap_tagging_level == M_HEAP_TAGGING_LEVEL_NONE) { error_log( "SetHeapTaggingLevel: re-enabling tagging after it was disabled is not supported"); - return false; + } else { + error_log("SetHeapTaggingLevel: switching between TBI and ASYNC is not supported"); } - break; + return false; default: error_log("SetHeapTaggingLevel: unknown tagging level"); return false; @@ -83,8 +97,16 @@ bool SetHeapTaggingLevel(void* arg, size_t arg_size) { heap_tagging_level = tag_level; info_log("SetHeapTaggingLevel: tag level set to %d", tag_level); - if (heap_tagging_level == M_HEAP_TAGGING_LEVEL_NONE && __libc_globals->heap_pointer_tag != 0) { - __libc_globals.mutate([](libc_globals* globals) { globals->heap_pointer_tag = 0; }); + if (heap_tagging_level == M_HEAP_TAGGING_LEVEL_NONE) { +#if defined(USE_SCUDO) + scudo_malloc_disable_memory_tagging(); +#endif + __libc_globals.mutate([](libc_globals* globals) { + // Preserve the untag mask (we still want to untag pointers when passing them to the + // allocator if we were doing so before), but clear the fixed tag and the check mask, + // so that pointers are no longer tagged and checks no longer happen. + globals->heap_pointer_tag &= 0xffull << UNTAG_SHIFT; + }); } return true; diff --git a/libc/bionic/malloc_tagged_pointers.h b/libc/bionic/malloc_tagged_pointers.h index 9c2a89bd1..212459bac 100644 --- a/libc/bionic/malloc_tagged_pointers.h +++ b/libc/bionic/malloc_tagged_pointers.h @@ -47,44 +47,62 @@ // rely on the implementation-defined value of this pointer tag, as it may // change. static constexpr uintptr_t POINTER_TAG = 0x3C; +static constexpr unsigned UNTAG_SHIFT = 40; +static constexpr unsigned CHECK_SHIFT = 48; static constexpr unsigned TAG_SHIFT = 56; #if defined(__aarch64__) static constexpr uintptr_t ADDRESS_MASK = (static_cast(1) << TAG_SHIFT) - 1; static constexpr uintptr_t TAG_MASK = static_cast(0xFF) << TAG_SHIFT; + +static inline uintptr_t FixedPointerTag() { + return __libc_globals->heap_pointer_tag & TAG_MASK; +} + +static inline uintptr_t PointerCheckMask() { + return (__libc_globals->heap_pointer_tag << (TAG_SHIFT - CHECK_SHIFT)) & TAG_MASK; +} + +static inline uintptr_t PointerUntagMask() { + return ~(__libc_globals->heap_pointer_tag << (TAG_SHIFT - UNTAG_SHIFT)); +} #endif // defined(__aarch64__) // Return a forcibly-tagged pointer. static inline void* TagPointer(void* ptr) { #if defined(__aarch64__) - return reinterpret_cast(reinterpret_cast(ptr) | - reinterpret_cast(__libc_globals->heap_pointer_tag)); + return reinterpret_cast(reinterpret_cast(ptr) | FixedPointerTag()); #else async_safe_fatal("Attempting to tag a pointer (%p) on non-aarch64.", ptr); #endif } -#if defined(__aarch64__) && !__has_feature(hwaddress_sanitizer) +#if defined(__aarch64__) // Return a forcibly-untagged pointer. The pointer tag is not checked for // validity. static inline void* UntagPointer(const volatile void* ptr) { return reinterpret_cast(reinterpret_cast(ptr) & ADDRESS_MASK); } -static void* SlowPathPointerCheck(const volatile void* ptr) { - uintptr_t ptr_tag = reinterpret_cast(ptr) & TAG_MASK; - uintptr_t heap_tag = reinterpret_cast(__libc_globals->heap_pointer_tag); +// Untag the pointer, and check the pointer tag iff the kernel supports tagged pointers and the +// pointer tag isn't being used by HWASAN or MTE. If the tag is incorrect, trap. +static inline void* MaybeUntagAndCheckPointer(const volatile void* ptr) { + if (__predict_false(ptr == nullptr)) { + return nullptr; + } + + uintptr_t ptr_int = reinterpret_cast(ptr); // Applications may disable pointer tagging, which will be propagated to // libc in the zygote. This means that there may already be tagged heap // allocations that will fail when checked against the zero-ed heap tag. The - // check bellow allows us to turn *off* pointer tagging and still allow - // tagged heap allocations to be freed, as long as they're using *our* tag. - if (__predict_false(heap_tag != 0 || ptr_tag != (POINTER_TAG << TAG_SHIFT))) { + // check below allows us to turn *off* pointer tagging (by setting PointerCheckMask() and + // FixedPointerTag() to zero) and still allow tagged heap allocations to be freed. + if ((ptr_int & PointerCheckMask()) != FixedPointerTag()) { // TODO(b/145604058) - Upstream tagged pointers documentation and provide // a link to it in the abort message here. async_safe_fatal("Pointer tag for %p was truncated.", ptr); } - return UntagPointer(ptr); + return reinterpret_cast(ptr_int & PointerUntagMask()); } // Return a tagged pointer iff the kernel supports tagged pointers, and `ptr` is @@ -96,23 +114,7 @@ static inline void* MaybeTagPointer(void* ptr) { return ptr; } -// Untag the pointer, and check the pointer tag iff the kernel supports tagged -// pointers. If the tag is incorrect, trap. -static inline void* MaybeUntagAndCheckPointer(const volatile void* ptr) { - if (__predict_false(ptr == nullptr)) { - return nullptr; - } - - uintptr_t ptr_tag = reinterpret_cast(ptr) & TAG_MASK; - uintptr_t heap_tag = reinterpret_cast(__libc_globals->heap_pointer_tag); - - if (__predict_false(heap_tag != ptr_tag)) { - return SlowPathPointerCheck(ptr); - } - return UntagPointer(ptr); -} - -#else // defined(__aarch64__) && !__has_feature(hwaddress_sanitizer) +#else // defined(__aarch64__) static inline void* UntagPointer(const volatile void* ptr) { return const_cast(ptr); } @@ -125,4 +127,4 @@ static inline void* MaybeUntagAndCheckPointer(const volatile void* ptr) { return const_cast(ptr); } -#endif // defined(__aarch64__) && !__has_feature(hwaddress_sanitizer) +#endif // defined(__aarch64__) diff --git a/libc/platform/bionic/malloc.h b/libc/platform/bionic/malloc.h index 99eefa475..0ea7e3c12 100644 --- a/libc/platform/bionic/malloc.h +++ b/libc/platform/bionic/malloc.h @@ -114,6 +114,8 @@ enum HeapTaggingLevel { // Address-only tagging. Heap pointers have a non-zero tag in the most significant byte which is // checked in free(). Memory accesses ignore the tag. M_HEAP_TAGGING_LEVEL_TBI = 1, + // Enable heap tagging if supported, at a level appropriate for asynchronous memory tag checks. + M_HEAP_TAGGING_LEVEL_ASYNC = 2, }; // Manipulates bionic-specific handling of memory allocation APIs such as diff --git a/libc/platform/bionic/mte_kernel.h b/libc/platform/bionic/mte_kernel.h index 804311cc5..2c777c949 100644 --- a/libc/platform/bionic/mte_kernel.h +++ b/libc/platform/bionic/mte_kernel.h @@ -48,4 +48,7 @@ #define PR_MTE_EXCL_SHIFT 3 #define PR_MTE_EXCL_MASK (0xffffUL << PR_MTE_EXCL_SHIFT) +#define SEGV_MTEAERR 6 +#define SEGV_MTESERR 7 + #endif diff --git a/tests/malloc_test.cpp b/tests/malloc_test.cpp index 4da6d3f4d..594441487 100644 --- a/tests/malloc_test.cpp +++ b/tests/malloc_test.cpp @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include #include #include @@ -40,7 +42,10 @@ #if defined(__BIONIC__) +#include "SignalUtils.h" + #include "platform/bionic/malloc.h" +#include "platform/bionic/mte_kernel.h" #include "platform/bionic/reserved_signals.h" #include "private/bionic_config.h" @@ -1196,3 +1201,70 @@ TEST(android_mallopt, set_allocation_limit_multiple_threads) { GTEST_SKIP() << "bionic extension"; #endif } + +#if defined(__BIONIC__) && defined(__aarch64__) && defined(ANDROID_EXPERIMENTAL_MTE) +template void CheckSiCode(int, siginfo_t* info, void*) { + if (info->si_code != SiCode) { + _exit(2); + } + _exit(1); +} + +static bool SetTagCheckingLevel(int level) { + int tagged_addr_ctrl = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0); + if (tagged_addr_ctrl < 0) { + return false; + } + + tagged_addr_ctrl = (tagged_addr_ctrl & ~PR_MTE_TCF_MASK) | level; + return prctl(PR_SET_TAGGED_ADDR_CTRL, tagged_addr_ctrl, 0, 0, 0) == 0; +} +#endif + +TEST(android_mallopt, tag_level) { +#if defined(__BIONIC__) && defined(__aarch64__) && defined(ANDROID_EXPERIMENTAL_MTE) + if (!(getauxval(AT_HWCAP2) & HWCAP2_MTE)) { + GTEST_SKIP() << "requires MTE support"; + return; + } + + std::unique_ptr p = std::make_unique(4); + + // First, check that memory tagging is enabled and the default tag checking level is async. + // We assume that scudo is used on all MTE enabled hardware; scudo inserts a header with a + // mismatching tag before each allocation. + EXPECT_EXIT( + { + ScopedSignalHandler ssh(SIGSEGV, CheckSiCode, SA_SIGINFO); + p[-1] = 42; + }, + testing::ExitedWithCode(1), ""); + + EXPECT_TRUE(SetTagCheckingLevel(PR_MTE_TCF_SYNC)); + EXPECT_EXIT( + { + ScopedSignalHandler ssh(SIGSEGV, CheckSiCode, SA_SIGINFO); + p[-1] = 42; + }, + testing::ExitedWithCode(1), ""); + + EXPECT_TRUE(SetTagCheckingLevel(PR_MTE_TCF_NONE)); + volatile int oob ATTRIBUTE_UNUSED = p[-1]; + + HeapTaggingLevel tag_level = M_HEAP_TAGGING_LEVEL_TBI; + EXPECT_FALSE(android_mallopt(M_SET_HEAP_TAGGING_LEVEL, &tag_level, sizeof(tag_level))); + + tag_level = M_HEAP_TAGGING_LEVEL_NONE; + EXPECT_TRUE(android_mallopt(M_SET_HEAP_TAGGING_LEVEL, &tag_level, sizeof(tag_level))); + std::unique_ptr p2 = std::make_unique(4); + EXPECT_EQ(0u, reinterpret_cast(p2.get()) >> 56); + + tag_level = M_HEAP_TAGGING_LEVEL_ASYNC; + EXPECT_FALSE(android_mallopt(M_SET_HEAP_TAGGING_LEVEL, &tag_level, sizeof(tag_level))); + + tag_level = M_HEAP_TAGGING_LEVEL_NONE; + EXPECT_TRUE(android_mallopt(M_SET_HEAP_TAGGING_LEVEL, &tag_level, sizeof(tag_level))); +#else + GTEST_SKIP() << "arm64 only"; +#endif +} diff --git a/tests/tagged_pointers_test.cpp b/tests/tagged_pointers_test.cpp index 4a666c49b..56d10374d 100644 --- a/tests/tagged_pointers_test.cpp +++ b/tests/tagged_pointers_test.cpp @@ -18,6 +18,7 @@ #include #include "platform/bionic/malloc.h" +#include "platform/bionic/mte.h" #include "utils.h" #include @@ -39,6 +40,10 @@ TEST(tagged_pointers, check_tagged_pointer_dies) { } #ifdef __aarch64__ + if (mte_supported()) { + GTEST_SKIP() << "Tagged pointers are not used on MTE hardware."; + } + void *x = malloc(1); // Ensure that `x` has a pointer tag. @@ -51,6 +56,9 @@ TEST(tagged_pointers, check_tagged_pointer_dies) { EXPECT_TRUE(android_mallopt(M_SET_HEAP_TAGGING_LEVEL, &tag_level, sizeof(tag_level))); EXPECT_DEATH(free(untag_address(malloc(1))), "Pointer tag for 0x[a-zA-Z0-9]* was truncated"); + tag_level = M_HEAP_TAGGING_LEVEL_ASYNC; + EXPECT_FALSE(android_mallopt(M_SET_HEAP_TAGGING_LEVEL, &tag_level, sizeof(tag_level))); + x = malloc(1); void *y = malloc(1); // Disable heap tagging.