diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h
index bfc64869..283ca36f 100644
--- a/include/pybind11/cast.h
+++ b/include/pybind11/cast.h
@@ -985,8 +985,12 @@ struct copyable_holder_caster<
     using base::value;
 
     bool load(handle src, bool convert) {
-        return base::template load_impl<copyable_holder_caster<type, std::shared_ptr<type>>>(
-            src, convert);
+        if (base::template load_impl<copyable_holder_caster<type, std::shared_ptr<type>>>(
+                src, convert)) {
+            sh_load_helper.maybe_set_python_instance_is_alias(src);
+            return true;
+        }
+        return false;
     }
 
     explicit operator std::shared_ptr<type> *() {
@@ -1049,6 +1053,7 @@ struct copyable_holder_caster<
     void load_value(value_and_holder &&v_h) {
         if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             sh_load_helper.loaded_v_h = v_h;
+            sh_load_helper.was_populated = true;
             value = sh_load_helper.get_void_ptr_or_nullptr();
             return;
         }
@@ -1192,14 +1197,19 @@ struct move_only_holder_caster<
     }
 
     bool load(handle src, bool convert) {
-        return base::template load_impl<
-            move_only_holder_caster<type, std::unique_ptr<type, deleter>>>(src, convert);
+        if (base::template load_impl<
+                move_only_holder_caster<type, std::unique_ptr<type, deleter>>>(src, convert)) {
+            sh_load_helper.maybe_set_python_instance_is_alias(src);
+            return true;
+        }
+        return false;
     }
 
     void load_value(value_and_holder &&v_h) {
         if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             sh_load_helper.loaded_v_h = v_h;
             sh_load_helper.loaded_v_h.type = typeinfo;
+            sh_load_helper.was_populated = true;
             value = sh_load_helper.get_void_ptr_or_nullptr();
             return;
         }
diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h
index c0bac7fd..26498df6 100644
--- a/include/pybind11/detail/class.h
+++ b/include/pybind11/detail/class.h
@@ -9,8 +9,8 @@
 
 #pragma once
 
-#include "../attr.h"
-#include "../options.h"
+#include <pybind11/attr.h>
+#include <pybind11/options.h>
 
 #include <cassert>
 #include <unordered_map>
diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h
index 40470b6a..524eb3f6 100644
--- a/include/pybind11/detail/common.h
+++ b/include/pybind11/detail/common.h
@@ -730,6 +730,11 @@ struct instance {
     bool simple_instance_registered : 1;
     /// If true, get_internals().patients has an entry for this object
     bool has_patients : 1;
+// Cannot use PYBIND11_INTERNALS_VERSION >= 6 here without refactoring.
+#if PYBIND11_VERSION_MAJOR >= 3
+    /// If true, this Python object needs to be kept alive for the lifetime of the C++ value.
+    bool is_alias : 1;
+#endif
 
     /// Initializes all of the above type/values/holders data (but not the instance values
     /// themselves)
diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h
index 332e8563..cbad7db7 100644
--- a/include/pybind11/detail/internals.h
+++ b/include/pybind11/detail/internals.h
@@ -12,10 +12,10 @@
 #include "common.h"
 
 #if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
-#    include "../gil.h"
+#    include <pybind11/gil.h>
 #endif
 
-#include "../pytypes.h"
+#include <pybind11/pytypes.h>
 
 #include <exception>
 #include <mutex>
diff --git a/include/pybind11/detail/struct_smart_holder.h b/include/pybind11/detail/struct_smart_holder.h
index 6fdd95f4..ccd28971 100644
--- a/include/pybind11/detail/struct_smart_holder.h
+++ b/include/pybind11/detail/struct_smart_holder.h
@@ -136,7 +136,6 @@ struct smart_holder {
     bool vptr_is_external_shared_ptr : 1;
     bool is_populated : 1;
     bool is_disowned : 1;
-    bool pointee_depends_on_holder_owner : 1; // SMART_HOLDER_WIP: See PR #2839.
 
     // Design choice: smart_holder is movable but not copyable.
     smart_holder(smart_holder &&) = default;
@@ -146,8 +145,7 @@ struct smart_holder {
 
     smart_holder()
         : vptr_is_using_noop_deleter{false}, vptr_is_using_builtin_delete{false},
-          vptr_is_external_shared_ptr{false}, is_populated{false}, is_disowned{false},
-          pointee_depends_on_holder_owner{false} {}
+          vptr_is_external_shared_ptr{false}, is_populated{false}, is_disowned{false} {}
 
     bool has_pointee() const { return vptr != nullptr; }
 
diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h
index 452a25ea..af92dc11 100644
--- a/include/pybind11/detail/type_caster_base.h
+++ b/include/pybind11/detail/type_caster_base.h
@@ -9,9 +9,10 @@
 
 #pragma once
 
-#include "../gil.h"
-#include "../pytypes.h"
-#include "../trampoline_self_life_support.h"
+#include <pybind11/gil.h>
+#include <pybind11/pytypes.h>
+#include <pybind11/trampoline_self_life_support.h>
+
 #include "common.h"
 #include "descr.h"
 #include "dynamic_raw_ptr_cast_if_possible.h"
@@ -707,6 +708,15 @@ inline std::unique_ptr<T, D> unique_with_deleter(T *raw_ptr, std::unique_ptr<D>
 
 template <typename T>
 struct load_helper : value_and_holder_helper {
+    bool was_populated = false;
+    bool python_instance_is_alias = false;
+
+    void maybe_set_python_instance_is_alias(handle src) {
+        if (was_populated) {
+            python_instance_is_alias = reinterpret_cast<instance *>(src.ptr())->is_alias;
+        }
+    }
+
     static std::shared_ptr<T> make_shared_ptr_with_responsible_parent(T *raw_ptr, handle parent) {
         return std::shared_ptr<T>(raw_ptr, shared_ptr_parent_life_support(parent.ptr()));
     }
@@ -727,7 +737,7 @@ struct load_helper : value_and_holder_helper {
             throw std::runtime_error("Non-owning holder (load_as_shared_ptr).");
         }
         auto *type_raw_ptr = static_cast<T *>(void_raw_ptr);
-        if (hld.pointee_depends_on_holder_owner) {
+        if (python_instance_is_alias) {
             auto *vptr_gd_ptr = std::get_deleter<pybindit::memory::guarded_delete>(hld.vptr);
             if (vptr_gd_ptr != nullptr) {
                 std::shared_ptr<void> released_ptr = vptr_gd_ptr->released_ptr.lock();
@@ -781,7 +791,7 @@ struct load_helper : value_and_holder_helper {
 
         auto *self_life_support
             = dynamic_raw_ptr_cast_if_possible<trampoline_self_life_support>(raw_type_ptr);
-        if (self_life_support == nullptr && holder().pointee_depends_on_holder_owner) {
+        if (self_life_support == nullptr && python_instance_is_alias) {
             throw value_error("Alias class (also known as trampoline) does not inherit from "
                               "py::trampoline_self_life_support, therefore the ownership of this "
                               "instance cannot safely be transferred to C++.");
diff --git a/include/pybind11/eigen/matrix.h b/include/pybind11/eigen/matrix.h
index eafbca14..80e72815 100644
--- a/include/pybind11/eigen/matrix.h
+++ b/include/pybind11/eigen/matrix.h
@@ -9,7 +9,8 @@
 
 #pragma once
 
-#include "../numpy.h"
+#include <pybind11/numpy.h>
+
 #include "common.h"
 
 /* HINT: To suppress warnings originating from the Eigen headers, use -isystem.
diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h
index eb40658c..79df26e8 100644
--- a/include/pybind11/eigen/tensor.h
+++ b/include/pybind11/eigen/tensor.h
@@ -7,7 +7,8 @@
 
 #pragma once
 
-#include "../numpy.h"
+#include <pybind11/numpy.h>
+
 #include "common.h"
 
 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h
index d25dbbc3..accba3fe 100644
--- a/include/pybind11/pybind11.h
+++ b/include/pybind11/pybind11.h
@@ -2316,7 +2316,8 @@ class class_ : public detail::generic_type {
         }
         auto *uninitialized_location = std::addressof(v_h.holder<holder_type>());
         auto *value_ptr_w_t = v_h.value_ptr<type>();
-        bool pointee_depends_on_holder_owner
+        // Try downcast from `type` to `type_alias`:
+        inst->is_alias
             = detail::dynamic_raw_ptr_cast_if_possible<type_alias>(value_ptr_w_t) != nullptr;
         if (holder_void_ptr) {
             // Note: inst->owned ignored.
@@ -2326,14 +2327,12 @@ class class_ : public detail::generic_type {
                        uninitialized_location, value_ptr_w_t, value_ptr_w_t)) {
             if (inst->owned) {
                 new (uninitialized_location) holder_type(holder_type::from_raw_ptr_take_ownership(
-                    value_ptr_w_t, /*void_cast_raw_ptr*/ pointee_depends_on_holder_owner));
+                    value_ptr_w_t, /*void_cast_raw_ptr*/ inst->is_alias));
             } else {
                 new (uninitialized_location)
                     holder_type(holder_type::from_raw_ptr_unowned(value_ptr_w_t));
             }
         }
-        v_h.holder<holder_type>().pointee_depends_on_holder_owner
-            = pointee_depends_on_holder_owner;
         v_h.set_holder_constructed();
     }
 
diff --git a/include/pybind11/stl/filesystem.h b/include/pybind11/stl/filesystem.h
index 935d7948..c16a9ae5 100644
--- a/include/pybind11/stl/filesystem.h
+++ b/include/pybind11/stl/filesystem.h
@@ -4,11 +4,11 @@
 
 #pragma once
 
-#include "../pybind11.h"
-#include "../detail/common.h"
-#include "../detail/descr.h"
-#include "../cast.h"
-#include "../pytypes.h"
+#include <pybind11/cast.h>
+#include <pybind11/detail/common.h>
+#include <pybind11/detail/descr.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/pytypes.h>
 
 #include <string>