Skip to content

Commit

Permalink
Systematically replace py:: with pybind11::
Browse files Browse the repository at this point in the history
No unit test changes, due to special circumstances (role eliminations).

However, this CL was used in a PyCLIF-pybind11 TGP (cl/667456010, 2024-08-26):

http://tap/OCL:667456010:BASE:667610796:1724689781533:d2626456:

```
    571 Failing Targets
    147 Broken Targets
```

PiperOrigin-RevId: 668846116
  • Loading branch information
rwgk committed Sep 6, 2024
1 parent 6c0aa9c commit 53e0b17
Show file tree
Hide file tree
Showing 11 changed files with 182 additions and 138 deletions.
51 changes: 27 additions & 24 deletions clif/pybind11/classes.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,13 @@ def generate_from(
decl: ast_pb2.Decl, superclass_name: str,
trampoline_class_names: Set[str], codegen_info: utils.CodeGenInfo,
) -> Generator[str, None, None]:
"""Generates a complete py::class_<>.
"""Generates a complete pybind11::class_<>.
Args:
decl: Class declaration in proto format.
superclass_name: String name of the superclass.
trampoline_class_names: A Set of class names whose member functions
will be overriden in Python.
trampoline_class_names: A Set of class names whose member functions will be
overriden in Python.
codegen_info: The information needed to generate pybind11 code.
Yields:
Expand All @@ -57,7 +57,7 @@ def generate_from(
namespace = '::'.join(namespaces[:i + 1])
yield I + I + f'using namespace ::{namespace};'
class_name = f'{class_decl.name.native}_class'
definition = f'py::classh<{class_decl.name.cpp_name}'
definition = f'pybind11::classh<{class_decl.name.cpp_name}'
implicit_upcast_bases = []
if not class_decl.suppress_upcasts:
for base in class_decl.bases:
Expand Down Expand Up @@ -94,23 +94,23 @@ def generate_from(
if class_decl.HasField('docstring'):
definition += f', {function_lib.generate_docstring(class_decl.docstring)}'

# If we generate `py::dynamic_attr()` for the base class, we also need to
# generate `py::dynamic_attr()` for the derived class.
# If we generate `pybind11::dynamic_attr()` for the base class, we also need
# to generate `pybind11::dynamic_attr()` for the derived class.
enable_instance_dict = class_decl.enable_instance_dict
for base in class_decl.bases:
if (base.HasField('cpp_canonical_type') and
base.cpp_canonical_type in codegen_info.dynamic_attr_types):
enable_instance_dict = True
break
if _USE_PYTYPE_TYPE_AS_METACLASS:
definition += ', py::metaclass((PyObject*) &PyType_Type)'
definition += ', py::release_gil_before_calling_cpp_dtor()'
definition += ', pybind11::metaclass((PyObject*) &PyType_Type)'
definition += ', pybind11::release_gil_before_calling_cpp_dtor()'
if mi_bases:
definition += ', py::multiple_inheritance()'
definition += ', pybind11::multiple_inheritance()'
if enable_instance_dict:
definition += ', py::dynamic_attr()'
definition += ', pybind11::dynamic_attr()'
if class_decl.final:
definition += ', py::is_final()'
definition += ', pybind11::is_final()'
definition += ');'
yield I + I + definition

Expand Down Expand Up @@ -172,21 +172,23 @@ def generate_from(

if (not ctor_defined and class_decl.cpp_has_def_ctor and
(not class_decl.cpp_abstract or trampoline_generated)):
yield I + I + f'{class_name}.def(py::init<>());'
yield I + I + f'{class_name}.def(pybind11::init<>());'

for base in implicit_upcast_bases:
mangled = legacy_types.Mangle(base.cpp_canonical_type)
I2 = I + I # pylint: disable=invalid-name
yield I2 + f'{class_name}.def('
yield I2 + I2 + f'"as_{mangled}",'
yield I2 + I2 + f'[]({class_decl.name.cpp_name}* self) {{'
yield I2 + I2 + I2 + 'return py::capsule(static_cast<void *>(self));'
yield I2 + I2 + I2 + 'return pybind11::capsule(static_cast<void *>(self));'
yield I2 + I2 + '}'
yield I2 + ');'

if not reduce_or_reduce_ex_defined:
yield I + I + (f'{class_name}.def("__reduce_ex__",' +
' ::clif_pybind11::ReduceExImpl, py::arg("protocol")=-1);')
yield I + I + (
f'{class_name}.def("__reduce_ex__", ::clif_pybind11::ReduceExImpl,'
' pybind11::arg("protocol")=-1);'
)

yield I + '}'

Expand All @@ -196,11 +198,12 @@ def _generate_iterator(
) -> Generator[str, None, None]:
template_param = ''
if function_lib.has_bytes_return(func_decl):
template_param = '<py::return_value_policy::_return_as_bytes>'
template_param = '<pybind11::return_value_policy::_return_as_bytes>'
yield (
f'{class_name}.def("__iter__", []({class_decl.name.cpp_name} &s)'
f'{{ return py::make_iterator{template_param}(s.begin(), s.end()); }}, '
'py::keep_alive<0, 1>());')
f'{class_name}.def("__iter__", []({class_decl.name.cpp_name} &s) {{'
f' return pybind11::make_iterator{template_param}(s.begin(), s.end());'
' }, pybind11::keep_alive<0, 1>());'
)


def _generate_constructor(
Expand Down Expand Up @@ -280,7 +283,7 @@ def _generate_constructor_overload(
func_decl, release_gil=False,
first_unknown_default_index=first_unknown_default_index)
if func_decl.name.native == '__init__' and func_decl.is_extend_method:
yield f'{class_name}.def(py::init([]({params_with_types}) {{'
yield f'{class_name}.def(pybind11::init([]({params_with_types}) {{'
if function_lib.unknown_default_argument_needs_non_default_value(
params_list, first_unknown_default_index, first_unknown_default_param):
yield I + function_lib.generate_value_error_for_unknown_default_param(
Expand All @@ -291,15 +294,15 @@ def _generate_constructor_overload(
yield from p.preprocess()
func_keeps_gil = function_lib.func_keeps_gil(func_decl)
if not func_keeps_gil:
yield I + 'py::gil_scoped_release release_gil;'
yield I + 'pybind11::gil_scoped_release release_gil;'
yield I + f'return {func_decl.name.cpp_name}({params});'
yield f'}}), {function_suffix}'

elif func_decl.name.native == '__init__':
cpp_name = class_decl.name.cpp_name
if trampoline_generated:
cpp_name = utils.trampoline_name(class_decl)
yield f'{class_name}.def(py::init([]({params_with_types}) {{'
yield f'{class_name}.def(pybind11::init([]({params_with_types}) {{'
if function_lib.unknown_default_argument_needs_non_default_value(
params_list, first_unknown_default_index, first_unknown_default_param):
yield I + function_lib.generate_value_error_for_unknown_default_param(
Expand All @@ -310,7 +313,7 @@ def _generate_constructor_overload(
yield from p.preprocess()
func_keeps_gil = function_lib.func_keeps_gil(func_decl)
if not func_keeps_gil and params_with_types:
yield I + 'py::gil_scoped_release release_gil;'
yield I + 'pybind11::gil_scoped_release release_gil;'
yield I + (f'return std::make_unique<{cpp_name}>'
f'({params}).release();')
yield f'}}), {function_suffix}'
Expand All @@ -328,7 +331,7 @@ def _generate_constructor_overload(
yield from p.preprocess()
func_keeps_gil = function_lib.func_keeps_gil(func_decl)
if not func_keeps_gil:
yield I + 'py::gil_scoped_release release_gil;'
yield I + 'pybind11::gil_scoped_release release_gil;'
yield I + (f'return std::make_unique<{class_decl.name.cpp_name}>'
f'({params});')
yield f'}}, {function_suffix}'
10 changes: 6 additions & 4 deletions clif/pybind11/consts.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ def generate_from(class_name: str, const_decl: ast_pb2.ConstDecl):
# Legacy CLIF ignores postconversion for clif::char_ptr.
if (function_lib.is_bytes_type(const_decl.type)
and const_decl.type.cpp_type != '::clif::char_ptr'):
return_value_policy = ', py::return_value_policy::_return_as_bytes'
yield I + (f'{class_name}.attr("{const_decl.name.native}") = '
f'py::cast(static_cast<{const_decl.type.cpp_type}>('
f'{const_decl.name.cpp_name}){return_value_policy});')
return_value_policy = ', pybind11::return_value_policy::_return_as_bytes'
yield I + (
f'{class_name}.attr("{const_decl.name.native}") = '
f'pybind11::cast(static_cast<{const_decl.type.cpp_type}>('
f'{const_decl.name.cpp_name}){return_value_policy});'
)
4 changes: 2 additions & 2 deletions clif/pybind11/enums.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ def generate_from(class_name: str, enum_decl: ast_pb2.EnumDecl):
"""Generates enums."""
kind = 'Enum' if enum_decl.enum_class else 'IntEnum'
yield I + (
f'{class_name} += py::native_enum<{enum_decl.name.cpp_name}>'
f'("{enum_decl.name.native}", py::native_enum_kind::{kind})'
f'{class_name} += pybind11::native_enum<{enum_decl.name.cpp_name}>'
f'("{enum_decl.name.native}", pybind11::native_enum_kind::{kind})'
)
for i, member in enumerate(enum_decl.members):
s = I + I + f'.value("{member.native}", {member.cpp_name})'
Expand Down
8 changes: 4 additions & 4 deletions clif/pybind11/function.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,13 +92,13 @@ def _generate_overload_for_unknown_default_function(
```
m.def("add", [](int a, Arg b, int c) {
return add(a, b, c);
}, py::arg("a"), py::arg("b"), py::arg("c") = 3);
}, pybind11::arg("a"), pybind11::arg("b"), pybind11::arg("c") = 3);
m.def("add", [](int a, int c) { // -b
throw py::value_error("argument b needs a non-default value");
}, py::arg("a"), py::kw_only(), py::arg("c"));
throw pybind11::value_error("argument b needs a non-default value");
}, pybind11::arg("a"), pybind11::kw_only(), pybind11::arg("c"));
m.def("add", [](int a) { // -b, -c
return add(a);
}, py::arg("a"));
}, pybind11::arg("a"));
```
Args:
Expand Down
66 changes: 38 additions & 28 deletions clif/pybind11/function_lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def __init__(self, param: ast_pb2.ParamDecl, param_name: str,
ctype.startswith('::std::shared_ptr'))
self.is_ptr = ptype.cpp_raw_pointer or is_smart_ptr
if is_self_param:
self.cpp_type = 'py::object'
self.cpp_type = 'pybind11::object'
self.gen_name = f'{param_name}_py'
if self.is_ptr:
self.function_argument = param_name
Expand Down Expand Up @@ -146,9 +146,11 @@ def unknown_default_argument_needs_non_default_value(
def generate_value_error_for_unknown_default_param(
func_decl: ast_pb2.FuncDecl, first_unknown_default_param: ast_pb2.ParamDecl
) -> str:
return (f'throw py::value_error("{func_decl.name.native}() argument '
f'{first_unknown_default_param.name.native} needs a non-default '
'value");')
return (
f'throw pybind11::value_error("{func_decl.name.native}() argument '
f'{first_unknown_default_param.name.native} needs a non-default '
'value");'
)


def generate_index_combination_for_unknown_default_func_decl(
Expand Down Expand Up @@ -211,20 +213,23 @@ def generate_return_value_policy_for_type(
if len(return_value_policy_list) > 1:
return f'{{{return_value_policy_str}}}'
else:
return ('py::return_value_policy_pack(std::vector<'
f'py::return_value_policy_pack>({{{return_value_policy_str}}}), '
'py::return_value_policy::_clif_automatic)')
return (
'pybind11::return_value_policy_pack('
'std::vector<pybind11::return_value_policy_pack>('
f'{{{return_value_policy_str}}}),'
' pybind11::return_value_policy::_clif_automatic)'
)
else:
if param_type.lang_type == 'bytes':
return 'py::return_value_policy::_return_as_bytes'
return 'pybind11::return_value_policy::_return_as_bytes'
elif is_callable_arg:
if param_type.lang_type == 'object':
return 'py::return_value_policy::automatic_reference'
return 'py::return_value_policy::_clif_automatic'
return 'pybind11::return_value_policy::automatic_reference'
return 'pybind11::return_value_policy::_clif_automatic'
elif reference_internal:
return 'py::return_value_policy::reference_internal'
return 'pybind11::return_value_policy::reference_internal'
else:
return 'py::return_value_policy::_clif_automatic'
return 'pybind11::return_value_policy::_clif_automatic'


def generate_return_value_policy_for_func_decl_params(
Expand All @@ -247,7 +252,7 @@ def generate_return_value_policy_for_func_decl_params(
else:
return return_value_policy_str
else:
return 'py::return_value_policy::_clif_automatic'
return 'pybind11::return_value_policy::_clif_automatic'


def generate_function_suffixes(
Expand All @@ -261,12 +266,12 @@ def generate_function_suffixes(
if py_args:
suffix += f'{py_args}, '
if func_decl.name.native in operators.ALL_OPS:
suffix += 'py::is_operator(), '
suffix += 'py::return_value_policy::_clif_automatic'
suffix += 'pybind11::is_operator(), '
suffix += 'pybind11::return_value_policy::_clif_automatic'
if func_decl.docstring:
suffix += f', {generate_docstring(func_decl.docstring)}'
if release_gil and not func_decl.py_keep_gil:
suffix += ', py::call_guard<py::gil_scoped_release>()'
suffix += ', pybind11::call_guard<pybind11::gil_scoped_release>()'
suffix += ');'
return suffix

Expand Down Expand Up @@ -387,10 +392,11 @@ def generate_py_args(func_decl: ast_pb2.FuncDecl,
params_list.append(
_generate_py_arg_without_default(param, return_value_policy_pack)
)
# Insert `py::kw_only()` at the index of the first parameter with unknown
# default value so that pybind11 is not confused about which overload to use.
# Insert `pybind11::kw_only()` at the index of the first parameter with
# unknown default value so that pybind11 is not confused about which
# overload to use.
if first_unknown_default_index != -1 and params_list:
params_list.insert(first_unknown_default_index, 'py::kw_only()')
params_list.insert(first_unknown_default_index, 'pybind11::kw_only()')
operators.fix_py_args_for_operators_in_place(
func_decl, params_list)
return ', '.join(params_list)
Expand All @@ -399,27 +405,30 @@ def generate_py_args(func_decl: ast_pb2.FuncDecl,
def _generate_py_arg_with_default(
param: ast_pb2.ParamDecl, return_value_policy_pack: str
) -> str:
"""Generate `py::arg` for parameters with default value."""
"""Generate `pybind11::arg` for parameters with default value."""
if return_value_policy_pack:
if param.default_value == 'nullptr':
return (
f'py::arg("{param.name.cpp_name}")'
f'pybind11::arg("{param.name.cpp_name}")'
f'.policies({return_value_policy_pack}) = {param.default_value}'
)
else:
return (
f'py::arg("{param.name.cpp_name}")'
f'pybind11::arg("{param.name.cpp_name}")'
f'.policies({return_value_policy_pack}) = '
f'static_cast<{param.type.cpp_type}>({param.default_value})'
)
else:
if param.default_value == 'nullptr':
if param.type.cpp_type == _CPP_TYPE_PYOBJECT_PTR_FROM_MATCHER:
return f'py::arg("{param.name.cpp_name}") = py::nullptr_default_arg()'
return f'py::arg("{param.name.cpp_name}") = {param.default_value}'
return (
f'pybind11::arg("{param.name.cpp_name}") ='
' pybind11::nullptr_default_arg()'
)
return f'pybind11::arg("{param.name.cpp_name}") = {param.default_value}'
else:
return (
f'py::arg("{param.name.cpp_name}") = '
f'pybind11::arg("{param.name.cpp_name}") = '
f'static_cast<{param.type.cpp_type}>({param.default_value})'
)

Expand All @@ -429,10 +438,11 @@ def _generate_py_arg_without_default(
) -> str:
if return_value_policy_pack:
return (
f'py::arg("{param.name.cpp_name}").policies({return_value_policy_pack})'
f'pybind11::arg("{param.name.cpp_name}")'
f'.policies({return_value_policy_pack})'
)
else:
return f'py::arg("{param.name.cpp_name}")'
return f'pybind11::arg("{param.name.cpp_name}")'


def _generate_return_value_policy_pack_for_py_arg(
Expand All @@ -444,7 +454,7 @@ def _generate_return_value_policy_pack_for_py_arg(
param.type.callable
)
if policy:
return f'py::return_value_policy_pack({policy})'
return f'pybind11::return_value_policy_pack({policy})'
else:
return ''

Expand Down
Loading

0 comments on commit 53e0b17

Please sign in to comment.