Add __pyclif_codegen_mode__ to all PyCLIF-generated extension modules.

This is to enable replacing ad-hoc inspection of the generated extension module docstrings with more formal conditions, e.g.:

```
    if __pyclif_codegen_mode__ == "pybind11":
```

This will make it easier to discover such conditions. (Current company priorities (staffing) make it seem very likely that the two codegen modes will co-exist for an extended period of time.)

For completeness and easy reference:

* cl/591787852 will update the only use case outside third_party/clif & devtools/clif.

* cl/591787855 will make the PyCLIF-pybind11 generated docstrings identical to the PyCLIF-C-API generated docstrings.

PiperOrigin-RevId: 592058095

Author: rwgk

clif/pybind11/generator.py

@@ -190,6 +190,7 @@ def generate_from(self, ast: ast_pb2.AST):
                f'"{self._module_name}", nullptr, '
                f'&module_def_{self._module_name});')
     yield I + 'try {'
+    yield I + I + 'm.attr("__pyclif_codegen_mode__") = "pybind11";'
     for s in self._generate_import_modules(ast):
       yield I + s
     yield I + I + ('m.doc() = "CLIF-generated pybind11-based module for '

clif/python/gen.py

@@ -310,7 +310,7 @@ def InitFunction(doc, meth_ref, init, dict_):
   yield '};'
   yield ''
   yield 'PyObject* Init() {'
-  yield I+'PyObject* module = PyModule_Create(&Module);'
+  yield I+'PyObject* module = ModuleCreateAndSetPyClifCodeGenMode(&Module);'
   yield I+'if (!module) return nullptr;'
   init_needs_err = False
   for s in init:

clif/python/module_test.py

@@ -68,7 +68,7 @@ def testFuncInit(self):
     };
 
     PyObject* Init() {
-      PyObject* module = PyModule_Create(&Module);
+      PyObject* module = ModuleCreateAndSetPyClifCodeGenMode(&Module);
       if (!module) return nullptr;
       return module;
     }"""))
@@ -110,7 +110,7 @@ def testModConst(self):
         };
 
         PyObject* Init() {
-          PyObject* module = PyModule_Create(&Module);
+          PyObject* module = ModuleCreateAndSetPyClifCodeGenMode(&Module);
           if (!module) return nullptr;
           if (something_wrong) goto err;
           if (PyModule_AddObject(module, "ONE", Clif_PyObjFrom(static_cast<int>(kOne), {})) < 0) goto err;

clif/python/runtime.cc

@@ -533,4 +533,29 @@ void SetIsNotConvertibleError(PyObject* py_obj, const char* cpp_type) {
                ClassName(py_obj), ClassType(py_obj), cpp_type);
 }
 
+namespace {
+
+bool SetPyClifCodeGenMode(PyObject* module, const char* codegen_mode) {
+  PyObject* codegen_mode_py = PyUnicode_FromString(codegen_mode);
+  if (codegen_mode_py == nullptr) {
+    return false;
+  }
+  int stat = PyObject_SetAttrString(module, "__pyclif_codegen_mode__",
+                                    codegen_mode_py);
+  Py_DECREF(codegen_mode_py);
+  return (stat == 0);
+}
+
+}  // namespace
+
+PyObject* ModuleCreateAndSetPyClifCodeGenMode(PyModuleDef* module_def) {
+  PyObject* module = PyModule_Create(module_def);
+  if (!module) return nullptr;
+  if (!SetPyClifCodeGenMode(module, "c_api")) {
+    Py_DECREF(module);
+    return nullptr;
+  }
+  return module;
+}
+
 }  // namespace clif

clif/python/runtime.h

@@ -214,6 +214,8 @@ void LogFatalIfPythonErrorOccurred();
 // PyObjectTypeIsConvertible*() functions above.
 void SetIsNotConvertibleError(PyObject* py_obj, const char* cpp_type);
 
+PyObject* ModuleCreateAndSetPyClifCodeGenMode(PyModuleDef* module_def);
+
 }  // namespace clif
 
 #endif  // CLIF_PYTHON_RUNTIME_H_

clif/testing/python/const_char_ptr_test.py

@@ -26,7 +26,7 @@ def testReturnConstCharPtrAsStr(self):
 
   def testReturnConstCharPtrAsBytes(self):
     res = const_char_ptr.ReturnConstCharPtrAsBytes()
-    if 'pybind11' not in const_char_ptr.__doc__:
+    if const_char_ptr.__pyclif_codegen_mode__ == 'c_api':
       # BUG: Return value should be bytes but is str (Python 3).
       self.assertIsInstance(res, str)  # Should be bytes.
       self.assertEqual(res, 'string literal')  # Should be b'string literal'.

clif/testing/python/copy_move_types_custom_from_as_test.py

@@ -18,7 +18,7 @@
 from clif.testing.python import copy_move_types_library
 
 ENV_IX = 1
-if "pybind11" in tm.__doc__:
+if tm.__pyclif_codegen_mode__ == "pybind11":
   ENV_IX += 2
 
 

clif/testing/python/extend_init_test.py

@@ -48,12 +48,15 @@ def test_init_with_kwargs(self):
     self.assertEqual(obj.get_k(), expected_k)
 
   def test_constructor_return_nullptr(self):
-    if 'pybind11' in extend_init.__doc__:
-      self.skipTest(
-          'Pybind11 does not allow factory functions to return nullptr.')
-    obj = extend_init.TestCase3(1)
-    with self.assertRaises(ValueError):
-      obj.get_value()
+    if extend_init.__pyclif_codegen_mode__ == 'pybind11':
+      with self.assertRaisesRegex(
+          TypeError, r'^pybind11::init\(\): factory function returned nullptr$'
+      ):
+        extend_init.TestCase3(1)
+    else:
+      obj = extend_init.TestCase3(1)
+      with self.assertRaises(ValueError):
+        obj.get_value()
 
   def test_extend_init_without_default_constructor(self):
     expected_value = 0
@@ -78,9 +81,6 @@ def test_py_err_from_constructor(self):
     self.assertIsInstance(
         extend_init.TestPyErrFromConstructor(0),
         extend_init.TestPyErrFromConstructor)
-    if 'pybind11' in extend_init.__doc__:
-      self.skipTest(
-          'Pybind11 does not allow factory functions to return nullptr.')
     with self.assertRaises(ValueError) as ctx:
       extend_init.TestPyErrFromConstructor(1)
     self.assertEqual(str(ctx.exception), 'RaisedFromExtendInit')

clif/testing/python/extend_methods_test.py

@@ -213,7 +213,7 @@ def testConcreteExtend(self):
     self.assertEqual(s, 'getter:18:ConcreteHolder')
 
   def testVirtualExtend(self):
-    if 'pybind11' not in extend_methods.__doc__:
+    if extend_methods.__pyclif_codegen_mode__ == 'c_api':
       expected_doc = 'Added to VirtualBaseHolder.'
       self.assertEqual(extend_methods.VirtualBaseHolder.__doc__, expected_doc)
       self.assertEqual(extend_methods.VirtualDerivedHolder.__doc__,

clif/testing/python/iterator_test.py

@@ -47,7 +47,7 @@ def testR5(self):
   def test_weakref(self):
     r = iterator.Ring5()
     self.assertIsNotNone(weakref.ref(r)())
-    if 'pybind11' not in iterator.__doc__:
+    if iterator.__pyclif_codegen_mode__ == 'c_api':
       with self.assertRaises(TypeError) as ctx:
         weakref.ref(iter(r))
       self.assertIn('cannot create weak reference', str(ctx.exception))

clif/testing/python/lambda_expressions_test.py

@@ -127,7 +127,7 @@ def test_context_manager(self):
 
   def test_cpp_function_set_python_exception(self):
     expected_exception_type = (
-        ValueError if 'pybind11' in lambda_expressions.__doc__
+        ValueError if lambda_expressions.__pyclif_codegen_mode__ == 'pybind11'
         else SystemError)
     with self.assertRaises(expected_exception_type):
       lambda_expressions.python_exception_in_function()

clif/testing/python/mock_uninitializable_test.py

@@ -78,7 +78,7 @@ def __init__(self):
 class DerivedFromNoCtorTest(absltest.TestCase):
 
   def testInstantiate(self):
-    if "pybind11" in tm.__doc__:
+    if tm.__pyclif_codegen_mode__ == "pybind11":
       regex_expected = (
           r"mock_uninitializable\.NoCtor\.__init__\(\) must be called when"
           r" overriding __init__$"

clif/testing/python/pass_none_test.py

@@ -55,7 +55,7 @@ def testPassSharedPtrHolder(self):
     res = pass_none.pass_shared_ptr_holder(ih)
     self.assertEqual(res, 559)
 
-    if 'pybind11' in pass_none.__doc__:
+    if pass_none.__pyclif_codegen_mode__ == 'pybind11':
       self.assertEqual(pass_none.pass_shared_ptr_holder(None), 17)
     else:
       # The generated C API code does not support shared_ptr from None.

clif/testing/python/pyobject_ptr_test.py

@@ -81,7 +81,7 @@ def cb(pvh):
       self.assertEqual(cc(cb, PyValueHolder(20)).value, 100)
     for _ in range(1000):
       self.assertEqual(cc(cb, PyValueHolder("Two")).value, 101)
-    if "pybind11" in tst.__doc__:
+    if tst.__pyclif_codegen_mode__ == "pybind11":
       for _ in range(1000):
         with self.assertRaisesRegex(ValueError, r"^Unknown pvh\.value: 3\.0$"):
           cc(cb, PyValueHolder(3.0))

clif/testing/python/raw_pointer_return_member_function_test.py

@@ -25,7 +25,7 @@ def testPersistentHolder(self):
     self.assertEqual(h.vec_at(1).int_value, 24)
 
   def testTemporaryHolder(self):
-    if 'pybind11' not in tst.__doc__:
+    if tst.__pyclif_codegen_mode__ == 'c_api':
       self.skipTest(
           'Only PyCLIF-pybind11 handles this correctly'
           ' (avoids creating a dangling pointer).'

clif/testing/python/simple_type_conversions_test.py

@@ -22,7 +22,7 @@ class SimpleTypeConversions(absltest.TestCase):
   def testSignedCharManipulation(self):
     self.assertEqual(tm.SignedCharManipulation(2), 29)
     for inp in [-129, 128]:
-      if 'pybind11' in tm.__doc__:
+      if tm.__pyclif_codegen_mode__ == 'pybind11':
         with self.assertRaises(TypeError) as ctx:
           tm.SignedCharManipulation(inp)
         self.assertIn('incompatible function arguments.', str(ctx.exception))
@@ -37,7 +37,7 @@ def testSignedCharManipulation(self):
 
   def testUnsignedCharManipulation(self):
     self.assertEqual(tm.UnsignedCharManipulation(3), 39)
-    if 'pybind11' in tm.__doc__:
+    if tm.__pyclif_codegen_mode__ == 'pybind11':
       with self.assertRaises(TypeError) as ctx:
         tm.SignedCharManipulation(256)
       self.assertIn('incompatible function arguments.', str(ctx.exception))
@@ -55,7 +55,7 @@ def testPassUint32InRange(self):
     self.assertEqual(tm.PassUint32(2**32 - 1), '4294967295')
 
   def testPassUint32Negative(self):
-    if 'pybind11' in tm.__doc__:
+    if tm.__pyclif_codegen_mode__ == 'pybind11':
       with self.assertRaisesRegex(
           TypeError, r'PassUint32\(\): incompatible function arguments.*'
       ):
@@ -69,7 +69,7 @@ def testPassUint32Negative(self):
         tm.PassUint32(-1)
 
   def testPassUint32OutOfRange(self):
-    if 'pybind11' in tm.__doc__:
+    if tm.__pyclif_codegen_mode__ == 'pybind11':
       with self.assertRaisesRegex(
           TypeError, r'PassUint32\(\): incompatible function arguments.*'
       ):

clif/testing/python/slots_test.py

@@ -20,7 +20,11 @@
 
 
 def expected_exception():
-  return AttributeError if 'pybind11' in slots.__doc__ else NotImplementedError
+  return (
+      AttributeError
+      if slots.__pyclif_codegen_mode__ == 'pybind11'
+      else NotImplementedError
+  )
 
 
 class SlotsTest(absltest.TestCase):

clif/testing/python/smart_ptrs_test.py

@@ -109,7 +109,7 @@ def testInfiniteLoopPerformSP(self):
 
   def testInfiniteLoopPerformUP(self):
     # No leak (manually verified under cl/362141489).
-    if 'pybind11' in smart_ptrs.__doc__:
+    if smart_ptrs.__pyclif_codegen_mode__ == 'pybind11':
       return  # pybind11 raises a ValueError (exercised above).
     while True:
       self.assertEqual(smart_ptrs.PerformUP(Add(17, 23)), 40)
@@ -125,7 +125,7 @@ def testInfiniteLoopRunStashedSPOperation(self):
 
   def testInfiniteLoopRunStashedUPOperation(self):
     # No leak (manually verified under cl/362141489).
-    if 'pybind11' in smart_ptrs.__doc__:
+    if smart_ptrs.__pyclif_codegen_mode__ == 'pybind11':
       return  # pybind11 raises a ValueError (exercised above).
     while True:
       s = smart_ptrs.OperationStashUP()

clif/testing/python/std_containers_copy_move_test.py

@@ -20,7 +20,7 @@
 from clif.testing.python import std_containers_copy_move as tm
 
 ENV_IX = 1
-if "pybind11" in tm.__doc__:
+if tm.__pyclif_codegen_mode__ == "pybind11":
   ENV_IX += 2
 
 

clif/testing/python/std_containers_test.py

@@ -167,7 +167,7 @@ def testContainerWithCustomizedHash(self):
         std_containers.consume_unordered_set_customized_hash(my_set), 2)
 
   @absltest.skipIf(
-      'pybind11' not in std_containers.__doc__,
+      std_containers.__pyclif_codegen_mode__ == 'c_api',
       'Legacy PyCLIF does not accept None as smart pointers.')
   def testAcceptNoneAsSmartPointers(self):
     self.assertIsNone(std_containers.unique_ptr_vector_round_trip(None))

clif/testing/python/t1_test.py

@@ -66,7 +66,7 @@ def testFunctionDocstring(self):
     self.assertIn('spans multiple lines', t1.Sum3.__doc__)
 
   def testFunctionDocstringNoTrailingWhitespaces(self):
-    if 'pybind11' in t1.__doc__:
+    if t1.__pyclif_codegen_mode__ == 'pybind11':
       self.skipTest(
           'pybind11 automatically adds a trailing whitespace to function '
           'docstrings.')

clif/testing/python/t3_test.py

@@ -34,7 +34,7 @@ def testEnum(self):
     self.assertRaises(TypeError, t3.K().M, (5))
 
   @absltest.skipIf(
-      'pybind11' in t3.__doc__,
+      t3.__pyclif_codegen_mode__ == 'pybind11',
       'When building with pybind11, t3._New.__module__ is a Module object '
       'instead of a string.',
   )

clif/testing/python/top_level_pass_test.py

@@ -23,6 +23,7 @@ def testEmptyModule(self):
     self.assertRegexpMatches(  # pylint: disable=deprecated-method
         top_level_pass.__doc__,
         'CLIF-generated .*module for .*top_level_pass.clif')
+    self.assertIn(top_level_pass.__pyclif_codegen_mode__, ('c_api', 'pybind11'))
 
 
 if __name__ == '__main__':

clif/testing/python/type_caster_test.py

@@ -49,8 +49,8 @@ def test_pyobjas_only(self):
     self.assertEqual(type_caster.get_value_pyobjas_only(10), 13)
 
   @absltest.skipIf(
-      'pybind11' not in type_caster.__doc__,
-      'Legacy PyCLIF does not recognize `Pybind11Ignore` in '
+      type_caster.__pyclif_codegen_mode__ == 'c_api',
+      'PyCLIF-C-API does not recognize `Pybind11Ignore` in '
       '`CLIF use` comments')
   def test_pybind11_ignore(self):
     with self.assertRaises(TypeError):

clif/testing/python/virtual_funcs_test.py

@@ -84,7 +84,7 @@ def func(self):
 class VirtualTest(absltest.TestCase):
 
   @absltest.skipIf(
-      'pybind11' in virtual_funcs.__doc__,
+      virtual_funcs.__pyclif_codegen_mode__ == 'pybind11',
       'Currently pybind11 does not throw exceptions when initializing abstract'
       'classes.')
   def testInitAbstract(self):

clif/testing/python/virtual_py_cpp_mix_test.py

@@ -42,7 +42,7 @@ def testGetFromCppUniquePtrPassingPyDerived(self):
 
   def testCppDerivedGet(self):
     d = virtual_py_cpp_mix.CppDerived()
-    if 'pybind11' in virtual_py_cpp_mix.__doc__:
+    if virtual_py_cpp_mix.__pyclif_codegen_mode__ == 'pybind11':
       expected = 212  # NOT GOOD, but this will be fixed in the switch to ...
     else:
       expected = 101  # ... pybind11, because this result is correct.