Shrink NaN to canonical form

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+Improve shrinking of non-standard NaN float values (:issue:`4277`).

hypothesis-python/src/hypothesis/internal/conjecture/choice.py

@@ -417,7 +417,7 @@ def choice_to_index(choice: ChoiceT, kwargs: ChoiceKwargsT) -> int:
             to_order=intervals.index_from_char_in_shrink_order,
         )
     elif isinstance(choice, float):
-        sign = int(sign_aware_lte(choice, -0.0))
+        sign = int(math.copysign(1.0, choice) < 0)
         return (sign << 64) | float_to_lex(abs(choice))
     else:
         raise NotImplementedError

hypothesis-python/src/hypothesis/internal/conjecture/shrinking/common.py

@@ -107,39 +107,47 @@ def run(self):
             self.run_step()
         self.debug("COMPLETE")
 
-    def incorporate(self, value):
+    def consider(self, value):
         """Try using ``value`` as a possible candidate improvement.
 
-        Return True if it works.
+        Return True if self.current is canonically equal to value after the call, either because
+        the value was incorporated as an improvement or because it had that value already.
         """
         value = self.make_immutable(value)
+        if value == self.current:
+            # shortcut for the simple (non-canonical) equality case, not required for correctness
+            return True
+        self.debug(f"considering {value!r}")
+        canonical = self.make_canonical(value)
+        if canonical in self.__seen:
+            return canonical == self.make_canonical(self.current)
+        self.__seen.add(canonical)
         self.check_invariants(value)
         if not self.left_is_better(value, self.current):
-            if value != self.current and (value == value):
-                self.debug(f"Rejected {value!r} as worse than {self.current=}")
-            return False
-        if value in self.__seen:
+            self.debug(f"Rejected {value!r} as no better than {self.current=}")
             return False
-        self.__seen.add(value)
         if self.__predicate(value):
             self.debug(f"shrinking to {value!r}")
             self.changes += 1
             self.current = value
             return True
-        return False
+        else:
+            self.debug(f"Rejected {value!r} not satisfying predicate")
+            return False
 
-    def consider(self, value):
-        """Returns True if make_immutable(value) == self.current after calling
-        self.incorporate(value)."""
-        self.debug(f"considering {value}")
-        value = self.make_immutable(value)
-        if value == self.current:
-            return True
-        return self.incorporate(value)
+    def make_canonical(self, value):
+        """Convert immutable value into a canonical and hashable, but not necessarily equal,
+        representation of itself.
+
+        This representation is used only for tracking already-seen values, not passed to the
+        shrinker.
+
+        Defaults to just returning the (immutable) input value.
+        """
+        return value
 
     def make_immutable(self, value):
-        """Convert value into an immutable (and hashable) representation of
-        itself.
+        """Convert value into an immutable representation of itself.
 
         It is these immutable versions that the shrinker will work on.
 

hypothesis-python/src/hypothesis/internal/conjecture/shrinking/floats.py

@@ -14,24 +14,23 @@
 from hypothesis.internal.conjecture.floats import float_to_lex
 from hypothesis.internal.conjecture.shrinking.common import Shrinker
 from hypothesis.internal.conjecture.shrinking.integer import Integer
-from hypothesis.internal.floats import MAX_PRECISE_INTEGER
+from hypothesis.internal.floats import MAX_PRECISE_INTEGER, float_to_int
 
 
 class Float(Shrinker):
     def setup(self):
-        self.NAN = math.nan
         self.debugging_enabled = True
 
-    def make_immutable(self, f):
-        f = float(f)
+    def make_canonical(self, f):
         if math.isnan(f):
-            # Always use the same NAN so it works properly in self.seen
-            f = self.NAN
+            # Distinguish different NaN bit patterns, while making each equal to itself
+            # To avoid accidental collision w/ a valid (large) float, convert to str.
+            return hex(float_to_int(f))
         return f
 
     def check_invariants(self, value):
-        # We only handle positive floats because we encode the sign separately
-        # anyway.
+        # We only handle positive floats (including NaN) because we encode the sign
+        # separately anyway.
         assert not (value < 0)
 
     def left_is_better(self, left, right):

hypothesis-python/tests/quality/test_float_shrinking.py

@@ -10,7 +10,7 @@
 
 import pytest
 
-from hypothesis import example, given, strategies as st
+from hypothesis import example, given, seed, strategies as st
 from hypothesis.internal.compat import ceil
 
 from tests.common.debug import minimal
@@ -45,3 +45,23 @@ def test_shrinks_downwards_to_integers_when_fractional(b):
         ).filter(lambda x: int(x) != x)
     )
     assert g == b + 0.5
+
+
+@pytest.mark.parametrize("s", range(10))
+def test_shrinks_to_canonical_nan(s):
+    # Regression test for #4277. A more reliable and minimal example could probably be found.
+    @given(
+        st.lists(
+            st.just(0) | st.floats().filter(lambda a: a != a), min_size=2, max_size=2
+        )
+    )
+    @seed(s)
+    def sort_is_reversible(l):
+        # fmt: off
+        assert sorted(l, reverse=True) == list(reversed(sorted(l)))  # noqa: C413
+        # fmt: on
+
+    try:
+        sort_is_reversible()
+    except AssertionError as e:
+        assert "[0, nan]" in e.__notes__[0]