diff --git a/CHANGELOG.md b/CHANGELOG.md index 4abf0e8..f7a3886 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,19 +1,26 @@ -## v0.3 - XXXX-XX-XX +## v0.3 - 2020-05-19 ### New features -* Python: - * introduced `ObjectHandler`, compatible with CPython's `PyObject *` - * added functions `beginCriticalSection()` / `endCriticalSection()` +* `MutexLock`: + * Introduced `Guard` (`std::lock_guard`) +* `Python`: + * revisited the whole API to make it simpler to use and more stable: + * removed function `arguments()`. Arguments are passed straight as a vector or an initializer list to the `call()` function + * functions `tuple()` and `list()` take a vector or an initializer list as an argument. Ellipsis are no longer used. + * introduced `ObjectHandler`, silently compatible with CPython `PyObject *` based API + * added functions `beginCriticalSection()` / `endCriticalSection()` to define critical sections in multithreaded applications * added unit tests and improved the documentation. ### Fixes -* Python: fixed a multithreading issue +* `Python`: fixed a multithreading issue ### Compatibility breakers -* Python: `ValueRef` and `ArgsRef` have been replaced by `ObjectHandler`. Explicit use of htese types should be replaced in your code. +* `Python`: API has been completely revisited (see above), though its philosophy remains unchanged. Changes in your code are expected to be strightforward. +* `MutexLock`: renamed `queueLock()` in `lock()`. Your code should be adapted the same way. + ## v0.2 - 2020-05-15 diff --git a/README.md b/README.md index 2ffc90c..84d3eeb 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,8 @@ it has no standardized replacement so far. ## Installation from binaries Should you just wish to use Shlublulib as a development tool, the binary distribution can be downloaded from our website: -* [Linux x86 version](http://shlublulib.shlublu.org/dist/shlublulib-linux-v0.3.zip) ([PGP sig](http://shlublulib.shlublu.org/dist/shlublulib-linux-v0.3.zip.asc) - SHA-256: `4ce0c2cff649297bad791092c6988c9b6023db599efdbf37f949aa284d31dc24`) -* [Win64 x86 version](http://shlublulib.shlublu.org/dist/shlublulib-win64-v0.3.zip) ([PGP sig](http://shlublulib.shlublu.org/dist/shlublulib-win64-v0.3.zip.asc) - SHA-256: `2ce011a85a25b4b2fe3698318be82fb89fb4a08326eb766f9a1537a82de91420`) +* [Linux x86 version](http://shlublulib.shlublu.org/dist/shlublulib-linux-v0.3.zip) ([PGP sig](http://shlublulib.shlublu.org/dist/shlublulib-linux-v0.3.zip.asc) - SHA-256: `0ae00fb99fcc1e14d9b7bcf211ef7f1fd1f8e2af8db4e4897179f76e9644c8d2`) +* [Win64 x86 version](http://shlublulib.shlublu.org/dist/shlublulib-win64-v0.3.zip) ([PGP sig](http://shlublulib.shlublu.org/dist/shlublulib-win64-v0.3.zip.asc) - SHA-256: `6762bcb2d7c16ab68cef2dd03132476d7a7533b6d18b4c57aaa196aba26cda36`) These archives are signed with [my PGP key](https://keyserver.ubuntu.com/pks/lookup?search=shlublu%40yahoo.fr&fingerprint=on&op=index) and contain: * the library file to link to your client programs diff --git a/include/shlublu/async/MutexLock.h b/include/shlublu/async/MutexLock.h index 332b4db..32a61cf 100644 --- a/include/shlublu/async/MutexLock.h +++ b/include/shlublu/async/MutexLock.h @@ -18,13 +18,15 @@ namespace shlublu Typical examples of use @code - MutexLock lock; // shared by MyThreadedTransactionsManager instances + MutexLock locker; // shared by MyThreadedTransactionsManager instances // ... void MyThreadedTransactionsManager::transactionalEnsembleThatDoesNotSupportConcurrency(int someNumericParameter) { - lock.queueLock(); + // ... operations that support concurrency ... + + locker.lock(); // ... operations that do not support concurrency ... @@ -33,21 +35,58 @@ namespace shlublu singleTransaction(i); } - lock.unlock(); + locker.unlock(); + + // ... operations that support concurrency ... } void MyThreadedTransactionsManager::singleTransaction(int someNumericParameter) { - lock.queueLock(); + // ... operations that support concurrency ... + + locker.lock(); // ... operations that do not support concurrency ... - lock.unlock(); + locker.unlock(); + + // ... operations that support concurrency ... } @endcode */ class MutexLock { +public: + /** + RAII convenience wrapper. + + This wrapper guarantees that the mutex will be unlock at the moment its destructor will be called. Used in a function or method, this + makes `MutexLock::unlock()` to be called at `return` or `throw` time with no further manual handling. + + Based on the standard `std::lock_guard` implementation. + + @see std::lock_guard + + Example + @code + MutexLock locker; // let's assume this is a memeber of MyThreadedTransactionsManager + + // ... + + int MyThreadedTransactionsManager::methodThatDoesNotSupportConcurrency() + { + MutexLock::Guard(locker); + + int x(someCall()); + + // ... operations that do not support concurrency ... + + return x; + } + @endcode + */ + using Guard = std::lock_guard; + public: /** Copy constructor is deleted. @@ -74,14 +113,14 @@ class MutexLock public: /** - Blocking lock. + Locks the mutex once available. Queues until the mutex is available for locking. A lock is available if: */ - void queueLock(); + void lock(); /** Unlocks the mutex. @@ -94,9 +133,13 @@ class MutexLock void unlock(); private: + /// @cond INTERNAL + std::recursive_mutex mMutex; std::thread::id mOwnerId; int mLockLevel; + + /// @endcond }; } diff --git a/include/shlublu/binding/Python.h b/include/shlublu/binding/Python.h index ed2cf07..f5efec2 100644 --- a/include/shlublu/binding/Python.h +++ b/include/shlublu/binding/Python.h @@ -1,6 +1,7 @@ #pragma once /** @file + Main file of the Python module. Based on the CPython standard API, this module is intended to make Python integration easier. See Python namespace documentation for details. @@ -17,11 +18,11 @@ namespace shlublu { /** @namespace shlublu::Python - Helper functions wrapping the CPython standard API. + Helper classes and functions wrapping the CPython standard API. - Not all functions of the CPython API are wrapped here. ShlubluLib Python is intended to make the most common operations simpler without preventing - users to use advanced features of the official API when needed. In particular, this module focuses of making the references count handling simpler - and less error-prone than doing it manually as CPython requires. + Not all functions of the CPython API are wrapped here. shlublu::Python is intended to make the most common operations simpler without preventing + users from using advanced features from the CPython API when needed. In particular, this module focuses of making the references count handling simpler + and less error-prone than doing it manually as CPython requires. Requirements
This module uses the official `Python3.x` library. @@ -40,7 +41,7 @@ namespace shlublu Typical examples of use @attention About concurrency
- The Python interpreter is unique and shared by all the threads of the process, and so is its memory state, including imports and objects.
- All functions below support concurrent access thanks to the use of a MutexLock. However, groups of Python calls that have a dependency - relationship in your C++ code should be surrounded by `beginCriticalSection()` / `endCriticalSection()` to prevent any concurrent call - to the global interpreter to occur in the middle of the section they constitute and make your reesluts inconsistent.
+ The CPython interpreter is global and shared by all the threads of the process. So is its memory state, including imports and objects.
+ All functions of this module are thread-safe. However, + Should you need more isolation, you can use the CPython API to setup multiple interpreters knowing that there are caveats described in the documentation. One of this limitation is the use of - the `PyGILState_*()` API. shlublu::Python doesn't use this API so there is no issue in that respect.
+ the `PyGILState_*()` API. As shlublu::Python doesn't use this API there should not be any issue there.

Forking is ok as the child process receives a full copy of the memory of the parent process made at `fork()` time. Testing shows the interpreter supports this very well, providing a full isolation, as long as the fork operation is conducted from a mono-thread process. Should you wish to fork a multi-threads process @@ -169,18 +175,16 @@ namespace shlublu */ namespace Python { - using PathEntriesList = std::vector; /**< Path as a vector of strings. */ - - using RawCode = std::string; /**< Plain Python code. */ - using Program = std::vector; /**< Complete program. */ + using PathEntriesList = std::vector; /**< Path, as a vector of strings. */ + using ObjectHandlersList = std::vector; /**< Parameters list to pass to `call()` or to functions that create collections. */ - using ScopeRef = PyObject*; /**< A scope is either an imported mudule, a namespace or an instance of a class. */ - using CallableRef = PyObject*; /**< Functions and methods. */ + using ObjectPointer = PyObject*; /**< Pointer to scope objects (either imported modules or instances of a class) or callable objects (functions or methods). Conversion from and to ObjectHandler is silent. */ - using ObjectHandlersList = std::vector; + using RawCode = std::string; /**< Plain Python code. */ + using Program = std::vector; /**< Complete program. Typical use is one line per element. Indentation is under the responsability of the programmer. */ - extern const std::string moduleMain; /**< Main module ("__main__"). Imported automatically by `import()`. */ - extern const std::string moduleBuiltins; /**< Built-ins module ("builtins"). Imported automatically by `import()`. */ + extern const std::string moduleMain; /**< Main module ("__main__"). Imported automatically at `init()` time. */ + extern const std::string moduleBuiltins; /**< Built-ins module ("builtins"). Imported automatically at `init()` time. */ /** Check whether Python is initialized. @@ -191,11 +195,12 @@ namespace Python /** Initializes Python. - Should `pythonSysPath` differ between two calls not separated by a `shutdown()`, the second path is appended to the first. + Should `pythonSysPath` differ between two calls not separated by a `shutdown()`, elements of the second path that are not part of the first are appended. This function can be called several times in a row with no issue. - @param programName the name to give the interpreter. The `argv[0]` argument given to the `main()` function of your program is a preferred choice though not mandatory, see below + @param programName the name to give the interpreter. The `argv[0]` argument given to the `main()` function of your program + is a preferred choice though not mandatory (see below) @param pythonSysPath system path that will be appended to `sys.path` if not already part of it @see Py_SetProgramName() */ @@ -203,7 +208,7 @@ namespace Python /** Shuts down Python. - `shutdown()` cleans up all references to the various objects previously handled or created by using Python and calls `Py_Finalize()`. + Cleans up all references to the various objects previously handled or created by using Python and calls `Py_Finalize()`. This function can be called several times in a row with no issue even if Python has not been initialized in the first place. It is called through `atexit()`, which makes its explicit use optional. @@ -222,105 +227,28 @@ namespace Python void execute(RawCode const& code); /** - Executes code as a program. - The whole program is executed in a row and cannot be interrupted by another thread. + Executes a program. + The whole program is executed in a row and cannot be interrupted by other threads using the CPython interpreter. - @param program code to execute, typically splitted in lines. Intentation should be materialized by spaces or `\t`. Empty lines are permitted. + @param program code to execute, typically splitted by lines. Intentation should be materialized by spaces or `\t`. Empty lines are permitted. @exception BindingException if Python is not initialized or if any line of code causes an error at interpretation time. Example @code - const Python::Program program({ "def testFunc(x):", "\tprint(x + ' is now printed')", "testFunc('text to print')" }); - Python::execute(program); // prints "text to print is now printed" + Python::execute({ "def testFunc(x):", "\tprint(x + ' is now printed')", "testFunc('text to print')" }); // prints "text to print is now printed" @endcode */ void execute(Program const& program); - /** - Prevent other threads to access to the global interpreter. - - Using this function is relevant in multi-threaded applications when groups of Python calls have a dependency relationships: - concurrent calls to the interpreter occuring in the middle of the execution of such groups would lead the interpreter to crash. - - Calls to this function should be followed in the same thread by as many calls to `endCriticalSection()`. Not doing so is - a cause of deadlocks. - - @exception BindingException if Python is not initialized. - - Example - @code - Python::init("pythonBinding"); - - Python::execute(Python::Program({ "def inc(x):", "\treturn x + 1" })); - - const long nbIt(5000000L); - long x(0); - long y(0); - - const auto job - ( - [](long iterations) -> long - { - long v(0); - const auto inc(Python::callable(Python::moduleMain, "inc")); - - for (long i = 0; i < iterations; ++i) - { - Python::beginCriticalSection(); - const auto pyVal(Python::call(inc, Python::arguments(1, PyLong_FromLong(v)))); - - v = PyLong_AsLong(pyVal); - - Python::forgetArgument(pyVal); - Python::endCriticalSection(); - } - - return v; - } - ); - - std::thread tx - ( - [&x, &job, nbIt]() - { - x = job(nbIt); - } - ); - - y = job(nbIt); - - tx.join(); - - // Assert: x == y == nbIt - - Python::shutdown(); - @endcode - */ - void beginCriticalSection(); - - - /** - Allow other threads to access to the global interpreter. - - Calls to this function should match calls to `beginCriticalSection()` performed in the same thread. - - @exception BindingException if this call doesn't match a call to `beginCriticalSection()` performed in the same thread. - */ - void endCriticalSection(); - - - ////////// - // The returned values below are garbage collected automatically, though calling forgetArgument() is possible ahead. - /** Imports a module by its name. - Importing a same module several times makes the reference of the first import to be returned. + Modules can be imported several times in a row with no issue, the same pointer being returned each time. - The returned reference is under control of Python for garbage collection at `shutdown()` time only. + The returned pointer is under control of Python for garbage collection at `shutdown()` time only. @param moduleName the name of the module, dot-delimited should it be part of a package - @return a reference to the imported module + @return a pointer to the imported module @exception BindingException if Python is not initialized, if the module cannot be found or if is causes errors at interpretation time Example @@ -328,18 +256,18 @@ namespace Python const auto pathModule(Python::import("os.path")); @endcode */ - ScopeRef import(std::string const& moduleName); + ObjectPointer import(std::string const& moduleName); /** Retrieves a previously imported module. - Modules have to be imported by `import()` to be retrieved that way. Modules imported by `execute("import ")` cannot - be retrieved by this function. + Modules have to be imported by `import()` to be retrieved that way. Modules imported by `execute()` (as `execute("import ")`) are + not retrieved by this function. - The returned reference is under control of Python for garbage collection at `shutdown()` time only. + The returned pointer is under control of Python for garbage collection at `shutdown()` time only. @param moduleName the name of the module, dot-delimited should it be part of a package - @return a reference to the module + @return a pointer to the module @exception BindingException if Python is not initialized or if the module has not be previsouly imported by `import()` Example @@ -348,39 +276,38 @@ namespace Python const auto pathModule(Python::retrieve("os.path")); @endcode */ - ScopeRef module(std::string const& moduleName); + ObjectPointer module(std::string const& moduleName); /** - Retrieves an object by its name from a scope reference. + Retrieves an object by its name from a scope pointer. - The returned handler is under control of Python for garbage collection. - Such a garbage collection is triggered by several functions that all mention this in their documentation. + The returned handler is under control of Python for garbage collection. + Such a garbage collection is triggered by several functions. This is mentioned in their documentation. - @param scopeRef reference to the module, namespace or object the object to retrieve belongs to + @param scope pointer to the module, namespace or object the object to retrieve belongs to @param objectName the name of the object to retrieve @return a handler of the retrieved object @exception BindingException if Python is not initialized or if the object cannot be found Example @code - Python::execute(Python::Program({ "import fractions as frac", "f = frac.Fraction(3, 2)"})); + Python::execute(Python::Program({ "import fractions as f", "fraction = f.Fraction(3, 2)" })); - const auto fraction(Python::object(Python::moduleMain, "f")); + const auto fraction(Python::object(Python::moduleMain, "fraction")); const auto denominator(Python::object(fraction, "denominator")); std::cout << String::xtos(PyLong_AsLong(denominator)) << std::endl; @endcode */ - ObjectHandler const& object(ScopeRef scopeRef, std::string const& objectName); + ObjectHandler const& object(ObjectPointer scope, std::string const& objectName); /** Retrieves an object by its name from a module name. - Objects can be retrieved regardless the way they were created. The returned handler is under control of Python for garbage collection. - Such a garbage collection is triggered by several functions that all mention this in their documentation. + Such a garbage collection is triggered by several functions. This is mentioned in their documentation. @param moduleName the name of the module the object to retrieve belongs to @param objectName the name of the object to retrieve @@ -389,25 +316,25 @@ namespace Python Example @code - Python::execute(Python::Program({ "import fractions as frac", "f = frac.Fraction(3, 2)"})); + Python::execute(Python::Program({ "import fractions as f", "fraction = f.Fraction(3, 2)"})); - const auto fraction(Python::object(Python::moduleMain, "f")); + const auto fraction(Python::object(Python::moduleMain, "fraction")); @endcode */ ObjectHandler const& object(std::string const& moduleName, std::string const& objectName); /** - Retrieves a callable object (function or method) by its name from a scope reference. - Callable objects are less likely to change than non callable objects during the execution of the program. - For this reason, references to callable are memorized. Multiple calls to this functions for the same scope reference and - callable name make the first retrieved reference to be returned, unless `forceReload` is set to true. + Retrieves a callable object (function or method) by its name from a scope pointer. + Callable objects are less likely to change than non-callable objects during the execution of the program. + For this reason, pointers to callable are memorized. Multiple calls to this function with the same scope argument and + callable name make the pointer retrieved in the first place to be returned, unless `forceReload` is set to true. - The returned reference is under control of Python for garbage collection at `shutdown()` time only. + The returned pointer is under control of Python for garbage collection at `shutdown()` time only. - @param scopeRef reference to the module, namespace or object the callable object to retrieve belongs to + @param scope pointer to the module, namespace or object the callable object to retrieve belongs to @param callableName the name of the callable object to retrieve - @param forceReload if true, refresh the callable object reference before returning it and dispose of any previous version - @return a reference to the retrieved object + @param forceReload if true, refresh the callable object pointer before returning it and dispose of any previous version + @return a pointer to the retrieved callable object @exception BindingException if Python is not initialized, if the object cannot be found or if it is not callable Example @@ -415,23 +342,23 @@ namespace Python const auto mathModule(Python::import("math")); const auto fabs(Python::callable(mathModule, "fabs")); - std::cout << PyFloat_AsDouble(Python::call(fabs, Python::arguments(1, PyFloat_FromDouble(-42.5)))) << std::endl; + std::cout << PyFloat_AsDouble(Python::call(fabs, { PyFloat_FromDouble(-42.5) })) << std::endl; @endcode */ - CallableRef callable(ScopeRef scopeRef, std::string const& callableName, bool forceReload = false); + ObjectPointer callable(ObjectPointer scope, std::string const& callableName, bool forceReload = false); /** - Retrieves a callable object (function or method) by its name from a module name. - Callable objects are less likely to change than non callable objects during the execution of the program. - For this reason, references to callable are memorized. Multiple calls to this functions for the same module and - callable names make the first retrieved reference to be returned, unless `forceReload` is set to true. + Retrieves a callable object (function) by its name from a module name. + Callable objects are less likely to change than non-callable objects during the execution of the program. + For this reason, pointers to callable are memorized. Multiple calls to this function with the same scope argument and + callable name make the pointer retrieved in the first place to be returned, unless `forceReload` is set to true. - The returned reference is under control of Python for garbage collection at `shutdown()` time only. + The returned pointer is under control of Python for garbage collection at `shutdown()` time only. @param moduleName name of the module the callable object to retrieve belongs to @param callableName the name of the callable object to retrieve - @param forceReload if true, refresh the callable object reference before returning it and dispose of any previous version - @return a reference to the retrieved object + @param forceReload if true, refresh the callable object pointer before returning it and dispose of any previous version + @return a pointer to the retrieved callable object @exception BindingException if Python is not initialized, if the module has not been imported by `import()`, if the object cannot be found or if it is not callable Example @@ -439,170 +366,111 @@ namespace Python Python::import("math"); const auto fabs(Python::callable("math", "fabs")); - std::cout << PyFloat_AsDouble(Python::call(fabs, Python::arguments(1, PyFloat_FromDouble(-42.5)))) << std::endl; + std::cout << PyFloat_AsDouble(Python::call(fabs, { PyFloat_FromDouble(-42.5) })) << std::endl; @endcode */ - CallableRef callable(std::string const& moduleName, std::string const& callableName, bool forceReload = false); - - /** - Creates an arguments tuple to pass to `call()`. - This function steals references of the passed objects: without increasing their references counts at creation - time, it decreases them at destruction time. - Should these objects be still needed for later use, they should be passed through: -
    -
  • `keepArgument()` for objects that are under control of Python
    • -
  • `controlArgument()` for objects that are not under control
    • -
- - The returned handler is under control of Python for garbage collection. - Such a garbage collection is triggered by several functions that all mention this in their documentation. - - @param args ordered collection of ObjectHandlers. They can either be controlled by Python or implicitelt created from `PyObject`pointers obtained from CPython functions. - @return a handler of the created tuple object - @exception BindingException if Python is not initialized - - Example - @code - const auto args(Python::arguments( { Python::fromAscii("text to print") } )); - - Python::call(Python::callable(Python::moduleBuiltins, "print"), args); - @endcode - */ - ObjectHandler const& arguments(ObjectHandlersList const& args); + ObjectPointer callable(std::string const& moduleName, std::string const& callableName, bool forceReload = false); /** Calls a callable with the given arguments. - `argumentsObject` is a handler of a tuple, typically created by `arguments()`. - A tuple returned by `tuple()` or by direct calls to CPython can also be used as `argumentsObject` if: -
    -
  • control of this tuple has been given to Python using `controlArgument()`,
    • -
  • or `keepArguments` is true
    • -
- - Once the callable returns, the references count of `argumentsObject` is decreased by a call to `forgetArgument()` unless `keepArguments` is true. + Once the callable returns, the references counts of the elements of `args` are decreased unless `keepArguments` is true in which case none are decreased. + Should you wich to only keep some of the passed arguments, `keepArguments` should be set to `false` and `controlArgument()`/`keepArgument()` should be used + with the arguments to preserve. The returned handler is under control of Python for garbage collection. - Such a garbage collection is triggered by several functions that all mention this in their documentation. + Such a garbage collection is triggered by several functions. This is mentioned in their documentation. - @param callableObject reference to the callable object to call, typically returned by `callable()` - @param argumentsObject handler of the arguments tuple to pass, typically returned by `arguments()`. If the callable takes no argument either `nullptr` or an empty tuple is fine. - @param keepArguments if true, the references count of `argumentsObject` will not be decreased so it can be reused later on - @return a handler of the result of the call, that can be `None` should the callable return no value. - @exception BindingException if Python is not initialized, if `argumentsObject` is neither an empty `nullptr` nor a tuple, or if `argumentsObject` is not under control of Python + @param callableObject pointer to the callable object to call, typically returned by `callable()` + @param args arguments of the call. They can either be obtained from previous Python calls or from CPython calls. Empty if no aregument is required. + @param keepArguments if true, the references counts of the elements of `args` will not be decreased so they can be reused later on + @return a handler of the result of the call, which is `None` if the callable returns no value. + @exception BindingException if Python is not initialized, or if any issue occurs during the call Example @code const auto print(Python::callable(Python::moduleBuiltins, "print")); - const auto args(Python::arguments(1, Python::fromAscii("text to print"))); - Python::call(print, args); + Python::call(print, { Python::fromAscii("text to print") }); @endcode */ - ObjectHandler const& call(CallableRef callableObject, ObjectHandler argumentsObject = nullptr, bool keepArguments = false); - - ////////// - // The returned values below are NOT garbage collected as they are supposed to see their reference stolen by arguments(), tuple() or addList(). - // Call controlArgument() below manually if not the case. + ObjectHandler const& call(ObjectPointer callableObject, ObjectHandlersList const& args = {}, bool keepArguments = false); /** - Creates a tuple object. - This function steals references of the passed objects: without increasing their references counts at creation - time, it decreases them at destruction time. - Should these objects be still needed for later use, they should be passed through: -
    -
  • `keepArgument()` for objects that are under control of Python
    • -
  • `controlArgument()` for objects that are not under control
    • -
+ Creates a tuple object initialized with the given arguments. + This function steals a reference from each element of `args` unless `keepArguments` + is true in which case they are all preserved. + Should you wich to only preserve some of the passed arguments, `keepArguments` should be set to `false` and `controlArgument()`/`keepArgument()` should be used + with the arguments to preserve. - The typical use case of this function is to prepare arguments to be passed as parameters to other functions of Python. - To be consistent with the behaviour of the functions of CPython, the returned handler is NOT under control of Python for garbage collection. - Should a control by Python be needed or just more convenient, this reference should be passed to `controlArgument()` the same way as a - result from a CPython function would be. + The returned handler is under control of Python for garbage collection. + Such a garbage collection is triggered by several functions. This is mentioned in their documentation. - @param args ordered collection of ObjectHandlers. They can either be controlled by Python or implicitelt created from `PyObject`pointers obtained from CPython functions. + @param args initializer of the tuple. These arguments can either be obtained from previous Python calls or from CPython calls. Empty to create an empty tuple. @return a handler of the created tuple object @exception BindingException if Python is not initialized Example @code - const auto tuple(Python::tuple(2, PyLong_FromLong(42), Python::fromAscii("42"))); + const auto tuple(Python::tuple({ PyLong_FromLong(42), Python::fromAscii("42") })); // Prints the length of the tuple ("2") Python::call ( Python::callable(Python::moduleBuiltins, "print"), - Python::arguments - ( - 1, - Python::call - ( - Python::callable(Python::moduleBuiltins, "len"), - Python::arguments(1, tuple) - ) - ) + { + Python::call(Python::callable(Python::moduleBuiltins, "len"), { tuple }) + } ); @endcode */ - ObjectHandler tuple(ObjectHandlersList const& args); + ObjectHandler tuple(ObjectHandlersList const& args = {}, bool keepArguments = false); /** - Creates a list object. - This function steals references of the passed objects: without increasing their references counts at creation - time, it decreases them at destruction time. - Should these objects be still needed for later use, they should be passed through: -
    -
  • `keepArgument()` for objects that are under control of Python
    • -
  • `controlArgument()` for objects that are not under control
    • -
+ Creates a list object initialized with the given arguments. + This function steals a reference from each element of `args` unless `keepArguments` + is true in which case they are all preserved. + Should you wich to only preserve some of the passed arguments, `keepArguments` should be set to `false` and `controlArgument()`/`keepArgument()` should be used + with the arguments to preserve. - The typical use case of this function is to prepare arguments to be passed as parameters to other functions of Python. - To be consistent with the behaviour of the functions of CPython, the returned handler is NOT under control of Python for garbage collection. - Should a control by Python be needed or just more convenient, this reference should be passed to `controlArgument()` the same way as a - result from a CPython function would be. + The returned handler is under control of Python for garbage collection. + Such a garbage collection is triggered by several functions. This is mentioned in their documentation. - @param args ordered collection of ObjectHandlers. They can either be controlled by Python or implicitelt created from `PyObject`pointers obtained from CPython functions. + @param args initializer of the list. These arguments can either be obtained from previous Python calls or from CPython calls. Empty to create an empty list. @return a handler of the created list object @exception BindingException if Python is not initialized Example @code - const auto list(Python::list(2, PyLong_FromLong(42), Python::fromAscii("42"))); + const auto list(Python::list({ PyLong_FromLong(42), Python::fromAscii("42") })); - Python::addList(list, PyFloat_FromDouble(42.0)); - - // Prints the length of the list ("3") + // Prints the length of the list ("2") Python::call ( Python::callable(Python::moduleBuiltins, "print"), - Python::arguments - ( - 1, - Python::call - ( - Python::callable(Python::moduleBuiltins, "len"), - Python::arguments(1, list) - ) - ) + { + Python::call(Python::callable(Python::moduleBuiltins, "len"), { list }) + } ); @endcode */ - ObjectHandler list(ObjectHandlersList const& args); + ObjectHandler list(ObjectHandlersList const& args = {}, bool keepArguments = false); /** - Adds an object item to the end of a list. - This function steals references of the passed object: without increasing its references count at creation - time, it decreases it at destruction time. - Should this objects be still needed for later use, `keepArguments` should be set to `true`. - - @param list handler of the list object to add an item to. It can either be controlled by Python or implicitely created from a `PyObject`pointer obtained from CPython functions. - @param item handler of the object itam to add. It can either be controlled by Python or implicitely created from a `PyObject`pointer obtained from CPython functions. - @param keepArguments if true, the references count of `item` will not be decreased so it can be reused later on + Adds an item to the end of a list. + This function steals a reference from `item` unless `keepArg` is true. + + The returned handler is under control of Python for garbage collection. + Such a garbage collection is triggered by several functions. This is mentioned in their documentation. + + @param list handler of the list object to add an item to. It can either be obtained from a previous Python call or implied by a `PyObject *` pointer obtained from a CPython call. + @param item handler of the item to add. It can either be obtained from a previous Python call or implied by a `PyObject *` pointer obtained from a CPython call. + @param keepArg if true, no reference will be stolen from `item` @exception BindingException if Python is not initialized or if `list` is not a list Example @code - const auto list(Python::list(2, PyLong_FromLong(42), Python::fromAscii("42"))); + const auto list(Python::list({ PyLong_FromLong(42), Python::fromAscii("42") })); Python::addList(list, PyFloat_FromDouble(42.0)); @@ -610,27 +478,19 @@ namespace Python Python::call ( Python::callable(Python::moduleBuiltins, "print"), - Python::arguments - ( - 1, - Python::call - ( - Python::callable(Python::moduleBuiltins, "len"), - Python::arguments(1, list) - ) - ) + { + Python::call(Python::callable(Python::moduleBuiltins, "len"), { list }) + } ); @endcode */ - void addList(ObjectHandler list, ObjectHandler item, bool keepArguments = false); + void addList(ObjectHandler list, ObjectHandler item, bool keepArg = false); /** - Converts a string to a UTF-8 string object handler. + Converts a string to a UTF-8 CPython string object. - The typical use case of this function is to prepare arguments to be passed as parameters to other functions of Python. - To be consistent with the behaviour of the functions of CPython, the returned handler is NOT under control of Python for garbage collection. - Should a control by Python be needed or just more convenient, this reference should be passed to `controlArgument()` the same way as a - result from a CPython function would be. + The returned handler is under control of Python for garbage collection. + Such a garbage collection is triggered by several functions. This is mentioned in their documentation. @param str the string to convert @return a handler of the UTF-8 string object representation of `str` @@ -641,32 +501,20 @@ namespace Python Python::call ( Python::callable(Python::moduleBuiltins, "print"), - Python::arguments - ( - 1, - Python::fromAscii("Text to print") - ) + { Python::fromAscii("Text to print") } ); @endcode */ ObjectHandler fromAscii(std::string const& str); - ////////// - // Helper functions to handle ValueRef's returned by call(), object() or straight from CPython - /** - Converts a UTF-8 string object to a std::string. - - The typical use case of this function is to handle string results returned by `call()` or 'object()'. - Such results are under control of Python. However this function can also be used with UTF-8 string objects - obtained from functions of CPython and that are not under control of Python. + Converts a UTF-8 CPython string object to a std::string. + This function steals a reference from `utfStr` unless `keepArg` is true. - This function steals references of `wstr` unless `keepArgument` is true. - - @param wstr the UTF-8 string object to convert - @param keepArgument if true, the references count of `wstr` will not be decreased so it can be reused later on - @return a handler of the string representation of `wstr` - @exception BindingException if Python is not initialized or if `wstr` is not a UTF-8 string object + @param utfStr the UTF-8 string object to convert + @param keepArg if true, no reference will be stolen from `utfStr` + @return the std::string representation of `utfStr` + @exception BindingException if Python is not initialized or if `utfStr` is not a UTF-8 CPython string object Example @code @@ -677,18 +525,16 @@ namespace Python std::cout << text << std::endl; @endcode */ - std::string toAscii(ObjectHandler wstr, bool keepArgument = false); - - ////////// - // Helper functions to handle references + std::string toAscii(ObjectHandler utfStr, bool keepArg = false); /** - Prevents an object reference from being stolen. - This function increases the reference count of an object under control. This prevents reference-stealing functions from taking ownership of the object. + Prevents an object reference from being stolen. + This function increases the reference count of an object under control of Python. This prevents reference-stealing functions from disposing of a reference + that is owned by the caller. @param object a handler of the object to preserve. It shoud be under control of Python. @return a new handler of `object` - @exception BindingException if Python is not initialized or if `object` is not under control + @exception BindingException if Python is not initialized or if `object` is not under control of Python Example @code @@ -697,28 +543,27 @@ namespace Python const auto math(Python::import("math")); const auto pow(Python::callable(math, "pow")); - const auto pyX(Python::call(Python::callable(Python::moduleMain, "giveMeFive"))); + const auto objectToPreserve(Python::call(Python::callable(Python::moduleMain, "giveMeFive"))); // displays 1 5 25 for (int y = 0; y < 3; ++y) { - // Not calling keepArgument(pyX) but passing straight pyX would cause a crash as pyX would be garbage collected after first call - const auto res(Python::call(pow, Python::arguments(2, Python::keepArgument(pyX), PyLong_FromLong(y)))); + // Not calling 'keepArgument(objectToPreserve)' but passing straight 'objectToPreserve' would cause a crash as it would be garbage-collected after first call + const auto res(Python::call(pow, { Python::keepArgument(objectToPreserve), PyLong_FromLong(y) })); std::cout << PyFloat_AsDouble(res) << " "; } - // We still have a reference on pyX. We can get rid of it right now using Python::forgetArgument() or let Python::shutdown() dispose of it + // We still own a reference on 'objectToPreserve' here. We can reuse it, get rid of it right now using Python::forgetArgument() or let Python::shutdown() dispose of it @endcode */ ObjectHandler const& keepArgument(ObjectHandler const& object); - /** - Places an object under cntrol of Python. - This makes Python aware of this object and include it in its garbage collection process. + Places an CPython object under control of Python. + This makes Python aware of this object to include it to its references count handling process. - @param a handler object the object to be controlled, miplicitely created from a `PyObject` pointer obtained from CPython API + @param object a handler of the object to be controlled. Typical use it to have it implicitely created from a `PyObject *` pointer obtained from the CPython API @return a new handler of `object` @exception BindingException if Python is not initialized or if `object` is already under control @@ -727,19 +572,19 @@ namespace Python const auto math(Python::import("math")); const auto pow(Python::callable(math, "pow")); - const auto pyX(Python::controlArgument(PyLong_FromLong(5))); // Python controls this object obtained from CPython + const auto objectX(Python::controlArgument(PyLong_FromLong(5))); // Python takes control of this object obtained from CPython // displays 1 5 25 for (int y = 0; y < 3; ++y) { - // Not calling keepArgument(pyX) but passing straight pyX would cause a crash as pyX would be garbage collected after - // first call, and only controlled objects can be passed through Python::keepArgument() - const auto res(Python::call(pow, Python::arguments(2, Python::keepArgument(pyX), PyLong_FromLong(y)))); + // Not calling 'keepArgument(objectX)' but passing straight 'objectX' would cause a crash as it would be garbage-collected after + // first call, while only controlled objects can be passed through Python::keepArgument() + const auto res(Python::call(pow, { Python::keepArgument(objectX), PyLong_FromLong(y) })); std::cout << PyFloat_AsDouble(res) << " "; } - // We still have a reference on pyX. As it is under control we can let Python::shutdown() dispose of it + // We still own a reference on 'objectX'. We can reuse it, get rid of it right now using Python::forgetArgument() or let Python::shutdown() dispose of it @endcode */ ObjectHandler const& controlArgument(ObjectHandler object); // Give Python the ownership of the passed objet created from outside without changing its references count. Object should not be controlled already. @@ -749,7 +594,7 @@ namespace Python This function allows saving memory by decreasing the reference count of an object that is no longer used without waiting for `shutdown()` to do it. @param object a handler of the object to forget - @exception BindingException if Python is not initialized, if `object` is not under control, or if the references count of `object` is zero + @exception BindingException if Python is not initialized, if `object` is not under control, or if the references count of `object` is zero or less Example @code @@ -760,20 +605,109 @@ namespace Python // Prints 100000 random numbers between min and max included for (int i = 0; i < 100000; ++i) { - const auto result(Python::call(randint, Python::arguments(2, Python::keepArgument(min), Python::keepArgument(max)))); + const auto result(Python::call(randint, { Python::keepArgument(min), Python::keepArgument(max) })); std::cout << PyLong_AsLong(result) << std::endl; - // We get rid of result right away as we won't use it anymore. - // Otherwise, with many iterations, a large amount of memory could end up wasted waiting for Python::shutdown(). - // This can also have an impact on performances as the internal references table maintain by Python gets bigger. + // We get rid of 'result' right away as we won't use it anymore. + // Otherwise, with many iterations, a large amount of memory could end up wasted waiting for 'Python::shutdown()' to dispose of these references. + // In a lesser extent this might also have an impact on performances as the internal references table maintain by Python would get bigger and bigger. Python::forgetArgument(result); } - // We let Python::shutdown() dispose of the remaining references of min and max, that's ok. + // Let's say we leave 'Python::shutdown()' to dispose of the remaining references of 'min' and 'max', that's ok. @endcode */ void forgetArgument(ObjectHandler const& object); // Forgets an object under control after having decreased its references count + + /** + Enters a critical section, preventing other threads from entering any Python critical section. + + This function is relevant in multi-threaded applications, either to guarantee data consistency or to prevent concurrent calls to native or CPython functions from + causing a crash (see note regarding concurrency in the detailed description of this namespace). + + Calls to this function should be followed in the same thread by as many calls to `endCriticalSection()`. Not doing so is + a cause of deadlocks. + + @exception BindingException if Python is not initialized. + + Example + @code + Python::init("pythonBinding"); + + const long nbIt(15000L); + std::vector values; + + std::thread producer + ( + [nbIt, &values]() + { + Python::execute(Python::Program({ "def dbl(x):", "\treturn 2 * x" })); + const auto dbl(Python::callable(Python::moduleMain, "dbl")); + + for (long i = 0; i < nbIt; ++i) + { + Python::beginCriticalSection(); + + const auto res(Python::call(dbl, { PyLong_FromLong(i) })); + values.push_back(res); + + Python::endCriticalSection(); + } + } + ); + + std::thread consumer + { + [nbIt, &values]() + { + long expected(0); + + while (expected < (2 * nbIt)) + { + Python::beginCriticalSection(); + + if (values.size()) + { + const auto obj(*values.begin()); + + // do something with 'obj' + + Python::forgetArgument(obj); + values.erase(values.begin()); + + Python::endCriticalSection(); + + expected += 2; + } + else + { + Python::endCriticalSection(); + + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + } + } + }; + + producer.join(); + consumer.join(); + + Python::shutdown(); + @endcode + */ + void beginCriticalSection(); + + + /** + Exits a critical section, allowing other threads to enter a Python critical section. + + Calls to this function should match calls to `beginCriticalSection()` performed in the same thread. + + @exception BindingException if this call doesn't match a call to `beginCriticalSection()` performed in the same thread. + */ + void endCriticalSection(); + } } diff --git a/include/shlublu/binding/Python_BindingException.h b/include/shlublu/binding/Python_BindingException.h index c022f56..e631d85 100644 --- a/include/shlublu/binding/Python_BindingException.h +++ b/include/shlublu/binding/Python_BindingException.h @@ -1,5 +1,11 @@ #pragma once +/** @file + Subpart of the Python module. + + See Python namespace documentation for details. +*/ + #include diff --git a/include/shlublu/binding/Python_ObjectHandler.h b/include/shlublu/binding/Python_ObjectHandler.h index 4197b1c..0659469 100644 --- a/include/shlublu/binding/Python_ObjectHandler.h +++ b/include/shlublu/binding/Python_ObjectHandler.h @@ -1,5 +1,11 @@ #pragma once +/** @file + Subpart of the Python module. + + See ObjectHandler documentation for details. +*/ + #define PY_SSIZE_T_CLEAN #include @@ -10,32 +16,157 @@ namespace shlublu namespace Python { +/** + Handler of CPython object pointers used by Python to handle references counts. + An instance represents a specific use of a given CPython object and is identified by an ID unique to this use case. + + A given CPython object can be involved in several instances should it have several use cases at a + given point of time. + + Two copies of a same handler have the same object pointers and the same ID as they refer to the same use case. + + ID is never zero but for empty instances. + + Conversion from and to `PyObject *` is silent. + + @see PyObject + + Typical examples of use + @code + #include + + #include + + using namespace shlublu; + + int main(int, char* argv[]) + { + int retCode(0); + + try + { + Python::init(*argv); + Python::execute("myFloat = 5.5"); + Python::ObjectHandler const& myFloat(Python::object(Python::moduleMain, "myFloat")); // Type 'Python::ObjectHandler const&' is explicit for this example though it is best to use `const auto` + + std::cout << PyFloat_AsDouble(myFloat) << std::endl; // 'Python::ObjectHandler' converts silently to CPython's 'PyObject *' + + const double anotherFloat(3.3); + + Python::controlArgument(PyFloat_FromDouble(anotherFloat)); // 'Python::ObjectHandler' is created silently from the CPython's 'PyObject *' + + // Python::shutdown() will dispose of this reference to the Python object created from anotherFloat + } + catch (Python::BindingException const& e) + { + std::cerr << "ERROR: " << e.what() << std::endl; + retCode = -1; + } + + return retCode; + } + @endcode +*/ class ObjectHandler { public: + /** + Key hasher to be used with unordered containers such as `std::unordered_map` or `std::unordered_set`. + @see std::unordered_map + @see std::unordered_set + + Typical examples of use + @code + std::unordered_set objectHandlersSet; + @endcode + */ class Hasher { public: + /** + Returns the key hash of an instance of ObjectHandler. + Hash is based on `ObjectHandler::id()` only. + */ int64_t operator()(ObjectHandler const& key) const; }; public: + /** + Empty constructor. + Sets both object pointer and ID to zero. + */ ObjectHandler(); + + /** + Copy constructor. + Both object pointer and ID are copied. + + @param src the instance to be copied. + */ ObjectHandler(ObjectHandler const& src); + + /** + Move constructor. + Both object pointer and ID are moved. + + @param src the instance to move in. + */ ObjectHandler(ObjectHandler&& src) noexcept; + + /** + `PyObject*` constructor. + Typically used as an implicit constructor for objects returned by the CPython API. + An ID is allocated automatically at creation time. + + @param pyObj the CPython object pointer this instance will be assigned. + @see PyObject + */ ObjectHandler(PyObject* pyObj); + /** + Assignment operator. + + Should `src` be another instance, both object pointer and ID are copied. + Should src be an object pointer, this pointer is copied and an ID is allocated automatically by an implied ObjectHandler. + + @param src other instance or `PyObject *` + @return a reference to `*this` + */ ObjectHandler& operator = (ObjectHandler src) noexcept; + /** + Exchanges two instances. + Both object pointer and ID are swaped. + Should src be an object pointer, an implied ObjectHandler is created and assigned an ID. + + @param other other instance to swap with + */ void swap(ObjectHandler& other) noexcept; + /** + Returns the object pointer this handler encapsulates. + @return a CPython object pointer + */ PyObject* get() const; + + /** + Returns the ID of the use case represented by this handler. + @return a use case ID + */ uint64_t id() const; + /** + `PyObject* ` cast operator. + Typically used as an implicit cast operator to pass instances of ObjectHandler to functions of the CPython API. + + @return the CPython object pointer this handler encapsulates. + @see PyObject + */ operator PyObject* () const; private: + /// @cond INTERNAL PyObject* mPyObj; uint64_t mId; @@ -43,10 +174,26 @@ class ObjectHandler uint64_t nextId(); static uint64_t sSequence; + /// @endcond }; +/** + Equality operator. + Compare use cases represented by two instances ObjectHandler. Refers to the ID to do so as two identical ID should come with two identical object pointers while the reverse is not true. + + @param lhs left operand + @param rhs right operand +*/ bool operator == (ObjectHandler const& lhs, ObjectHandler const& rhs); + +/** + Inequality operator. + Compare use cases represented by two instances ObjectHandler. Refers to the ID to do so as two identical ID should come with two identical object pointers while the reverse is not true. + + @param lhs left operand + @param rhs right operand +*/ bool operator != (ObjectHandler const& lhs, ObjectHandler const& rhs); } diff --git a/include/shlublu/binding/Python_ObjectHandlersCollection.h b/include/shlublu/binding/Python_ObjectHandlersCollection.h new file mode 100644 index 0000000..a4ce470 --- /dev/null +++ b/include/shlublu/binding/Python_ObjectHandlersCollection.h @@ -0,0 +1,121 @@ +#pragma once + +/** @file + Subpart of the Python module. + + See ObjectHandlersCollection documentation for details. +*/ + +#include + +#include +#include + + +namespace shlublu +{ + +namespace Python +{ + +/** + Collection of ObjectHandler used internally by Python. + + Each element of a collection instance correspond to a unique ObjectHandler use case. +*/ +class ObjectHandlersCollection +{ +public: + /** + Definition of the storage container. + */ + using Storage = std::unordered_set; + +public: + /** + Empty constructor. + Creates an empty collection. + */ + ObjectHandlersCollection(); + + /** + Copy constructor is deleted. + */ + ObjectHandlersCollection(ObjectHandlersCollection const&) = delete; + + /** + Move constructor is deleted. + */ + ObjectHandlersCollection(ObjectHandlersCollection&&) = delete; + + /** + Registers a use case represented by an ObjectHandler. + + @param oh the use case to register + @return a reference to this use case once registered + @exception BindingException if `oh`'s ID is already registered + */ + ObjectHandler const& registerObject(ObjectHandler const& oh); + + /** + Unregisters a use case represented by an ObjectHandler. + + @param oh the use case to unregister + @exception BindingException if `oh`'s ID has not been registered or has been unregistered already + */ + void unregisterObject(ObjectHandler const& oh); + + /** + Clears the collection. + + All use cases are unregistered by a call to this function. + */ + void clear(); + + /** + Returns the registration status of a use case represented by an ObjectHandler. + + @param oh the use case to assess + @return the registration status of `oh` based on its ID + */ + bool isRegistered(ObjectHandler const& oh) const; + + /** + Returns the size of this collection. + + @return the number of elements it contains. + */ + size_t size() const; + + /** + Returns an iterator to the first element of this collection. + Order is internal to ObjectHandlersCollection. It does not follow the registration order. + Iterating from `begin()` to `end()` guarantees to go through each element once and only once. + + @return an iterator to the first element of the collection + */ + Storage::const_iterator begin() const; + + /** + Returns an iterator to the past-the-end element of this collection. + Order is internal to ObjectHandlersCollection. It does not follow the registration order. + Iterating from `begin()` to `end()` guarantees to go through each element once and only once. + + @return an iterator to the past-the-end element of the collection + */ + Storage::const_iterator end() const; + +private: + /// @cond INTERNAL + + Storage mObjects; + + /// @endcond +}; + + +} + +} + + diff --git a/shlublu-linux.vcxproj b/shlublu-linux.vcxproj index ad55d02..1ba8671 100644 --- a/shlublu-linux.vcxproj +++ b/shlublu-linux.vcxproj @@ -38,6 +38,7 @@ + @@ -46,6 +47,7 @@ + diff --git a/shlublu-linux.vcxproj.filters b/shlublu-linux.vcxproj.filters index a46716f..84535c0 100644 --- a/shlublu-linux.vcxproj.filters +++ b/shlublu-linux.vcxproj.filters @@ -54,6 +54,9 @@ src\binding + + src\binding + @@ -80,5 +83,8 @@ src\binding + + include\binding + \ No newline at end of file diff --git a/shlublu.vcxproj b/shlublu.vcxproj index ea306df..f1f762b 100644 --- a/shlublu.vcxproj +++ b/shlublu.vcxproj @@ -22,6 +22,7 @@ + @@ -30,6 +31,7 @@ + diff --git a/shlublu.vcxproj.filters b/shlublu.vcxproj.filters index ec9e0b4..cfaf2db 100644 --- a/shlublu.vcxproj.filters +++ b/shlublu.vcxproj.filters @@ -54,6 +54,9 @@ src\binding + + src\binding + @@ -80,5 +83,8 @@ include\binding + + include\binding + \ No newline at end of file diff --git a/src/async/MutexLock.cpp b/src/async/MutexLock.cpp index 556ae2b..5e55b7f 100644 --- a/src/async/MutexLock.cpp +++ b/src/async/MutexLock.cpp @@ -16,7 +16,7 @@ MutexLock::MutexLock(bool take) { if (take) { - queueLock(); + lock(); } } @@ -30,7 +30,7 @@ MutexLock::~MutexLock() } -void MutexLock::queueLock() +void MutexLock::lock() { mMutex.lock(); @@ -59,4 +59,5 @@ void MutexLock::unlock() } } + } diff --git a/src/binding/Python.cpp b/src/binding/Python.cpp index 5b765fe..03510eb 100644 --- a/src/binding/Python.cpp +++ b/src/binding/Python.cpp @@ -1,109 +1,89 @@ -#include -SHLUBLU_OPTIMIZE_OFF(); - #include -#include #include +#include #include #include -#include #ifdef _WIN32 #pragma warning( disable : 6285) #endif -// Useful documentation: -// https://docs.python.org/fr/3/extending/embedding.html#embedding-python-in-c -// https://docs.python.org/3/c-api/ - - -static shlublu::MutexLock __pythonLock(false); - -static wchar_t* __pythonArgv0(nullptr); - -static std::unordered_map __pythonModules; -static std::unordered_map> __pythonCallables; - -static std::unordered_set __pythonObjects; - +namespace shlublu +{ -static void __pythonGrab() +namespace Python { - __pythonLock.queueLock(); -} +static std::unordered_map __modules; +static std::unordered_map> __callables; +static ObjectHandlersCollection __sObjects; -static void __pythonRelease() -{ - __pythonLock.unlock(); -} +static wchar_t* __sArgv0(nullptr); +static MutexLock __sLock(false); -static void __pythonThrowException(std::string const& message) +static void __throwException(std::string const& message) { - __pythonRelease(); - - throw shlublu::Python::BindingException(message); + throw BindingException(message); } -static void __pythonShouldBeInitialized() +static void __shouldBeInitialized() { - __pythonGrab(); - - if (!__pythonArgv0) + if (!isInitialized()) { - __pythonThrowException("__pythonShouldBeInitialized(): not in initialized state."); + __throwException("__shouldBeInitialized(): not in initialized state."); } } -static void __pythonWithdrawAsCatched(shlublu::Python::ObjectHandler const& object) +static void __decRef(ObjectHandler const& object) { - if (__pythonObjects.count(object)) + if (object.get()->ob_refcnt < 1) { - __pythonObjects.erase(object); + __throwException("Python::__DecRef(): references count is already " + String::xtos(object.get()->ob_refcnt)); } + + Py_DECREF(object); } -static void __pythonDecRef(shlublu::Python::ObjectHandler const& object) +static void __handleObjectUnregistration(ObjectHandler const& object, bool keepArg) { - if (!object.get()->ob_refcnt) + if (keepArg) { - __pythonThrowException("Python::loseArgument(): references count is already zero"); + Py_INCREF(object); + } + else if (__sObjects.isRegistered(object)) + { + __sObjects.unregisterObject(object); } - - Py_DECREF(object); } -namespace shlublu -{ - -const std::string Python::moduleMain = "__main__"; -const std::string Python::moduleBuiltins = "builtins"; +const std::string moduleMain = "__main__"; +const std::string moduleBuiltins = "builtins"; -bool Python::isInitialized() +bool isInitialized() { - return __pythonArgv0 != nullptr; + return __sArgv0 != nullptr; } -void Python::init(std::string const& programName, PathEntriesList const& pathList) +void init(std::string const& programName, PathEntriesList const& pathList) { - __pythonGrab(); + MutexLock::Guard guard(__sLock); - if (__pythonArgv0 == nullptr) + if (__sArgv0 == nullptr) { - __pythonArgv0 = Py_DecodeLocale(programName.c_str(), nullptr); + __sArgv0 = Py_DecodeLocale(programName.c_str(), nullptr); - Py_SetProgramName(__pythonArgv0); + Py_SetProgramName(__sArgv0); Py_Initialize(); if (!PyEval_ThreadsInitialized()) @@ -123,64 +103,57 @@ void Python::init(std::string const& programName, PathEntriesList const& pathLis { execute("if '" + pathEntry + "' not in sys.path:\n\tsys.path.append('" + pathEntry + "')"); } - - __pythonRelease(); } -void Python::shutdown() +void shutdown() { - SHLUBLU_TODO("References count deserves a unit test suite"); - - __pythonGrab(); + MutexLock::Guard guard(__sLock); - if (__pythonArgv0) + if (__sArgv0) { - for (auto& callableMapEntry : __pythonCallables) + for (auto& callableMapEntry : __callables) { for (auto& callableEntry : callableMapEntry.second) { - __pythonDecRef(callableEntry.second); + __decRef(callableEntry.second); } } - __pythonCallables.clear(); + __callables.clear(); - for (auto& value : __pythonObjects) + for (auto& value : __sObjects) { - __pythonDecRef(value); + __decRef(value); } - __pythonObjects.clear(); + __sObjects.clear(); - for (auto& moduleEntry : __pythonModules) + for (auto& moduleEntry : __modules) { - __pythonDecRef(moduleEntry.second); + __decRef(moduleEntry.second); } - __pythonModules.clear(); + __modules.clear(); Py_Finalize(); - PyMem_RawFree(__pythonArgv0); + PyMem_RawFree(__sArgv0); - __pythonArgv0 = nullptr; + __sArgv0 = nullptr; } - - __pythonRelease(); } -void Python::execute(RawCode const& code) +void execute(RawCode const& code) { - __pythonShouldBeInitialized(); + MutexLock::Guard guard(__sLock); + __shouldBeInitialized(); if (PyRun_SimpleString(code.c_str()) < 0) { - __pythonThrowException("Python::execute(): Instruction '" + code + "' caused an error"); + __throwException("Python::execute(): Instruction '" + code + "' caused an error"); } - - __pythonRelease(); } -void Python::execute(Program const& program) +void execute(Program const& program) { RawCode code; @@ -193,354 +166,314 @@ void Python::execute(Program const& program) } -void Python::beginCriticalSection() +void beginCriticalSection() { - __pythonShouldBeInitialized(); + __sLock.lock(); + + try + { + __shouldBeInitialized(); + } + catch (std::exception const& e) + { + __sLock.unlock(); + throw e; + } } -void Python::endCriticalSection() +void endCriticalSection() { - __pythonRelease(); + __sLock.unlock(); } -Python::ScopeRef Python::import(std::string const& moduleName) +ObjectPointer import(std::string const& moduleName) { - __pythonShouldBeInitialized(); + MutexLock::Guard guard(__sLock); + __shouldBeInitialized(); - ScopeRef pythonModule(nullptr); + ObjectPointer pythonModule(nullptr); - if (!__pythonModules.count(moduleName)) + if (!__modules.count(moduleName)) { const auto pythonModuleName(PyUnicode_DecodeFSDefault(moduleName.c_str())); pythonModule = PyImport_Import(pythonModuleName); - __pythonDecRef(pythonModuleName); + __decRef(pythonModuleName); if (!pythonModule) { - __pythonThrowException("Python::import(): Cannot import module '" + moduleName + "'"); + __throwException("Python::import(): Cannot import module '" + moduleName + "'"); } - __pythonModules.emplace(moduleName, pythonModule); + __modules.emplace(moduleName, pythonModule); } else { - pythonModule = __pythonModules.at(moduleName); + pythonModule = __modules.at(moduleName); } - __pythonRelease(); - return pythonModule; } -Python::ScopeRef Python::module(std::string const& moduleName) +ObjectPointer module(std::string const& moduleName) { - __pythonShouldBeInitialized(); + MutexLock::Guard guard(__sLock); + __shouldBeInitialized(); - if (!__pythonModules.count(moduleName)) + if (!__modules.count(moduleName)) { - __pythonThrowException("Python::module(): Cannot retrieve '" + moduleName + "' in imported modules"); + __throwException("Python::module(): Cannot retrieve '" + moduleName + "' in imported modules"); } - auto const& ret(__pythonModules.at(moduleName)); - - __pythonRelease(); + auto const& ret(__modules.at(moduleName)); return ret; } -Python::ObjectHandler const& Python::object(ScopeRef scopeRef, std::string const& objectName) +ObjectHandler const& object(ObjectPointer scope, std::string const& objectName) { - __pythonShouldBeInitialized(); + MutexLock::Guard guard(__sLock); + __shouldBeInitialized(); - const ObjectHandler pythonObject(PyObject_GetAttrString(scopeRef, objectName.c_str())); + const ObjectHandler pythonObject(PyObject_GetAttrString(scope, objectName.c_str())); if (!pythonObject.get()) { - __pythonThrowException("Python::object(): Cannot access to object '" + objectName + "'"); + __throwException("Python::object(): Cannot access to object '" + objectName + "'"); } - const auto& ret(*__pythonObjects.emplace(pythonObject).first); - - __pythonRelease(); - - return ret; + return __sObjects.registerObject(pythonObject); } -Python::ObjectHandler const& Python::object(std::string const& moduleName, std::string const& objectName) +ObjectHandler const& object(std::string const& moduleName, std::string const& objectName) { return object(module(moduleName), objectName); } -Python::CallableRef Python::callable(ScopeRef scopeRef, std::string const& callableName, bool forceReload) +ObjectPointer callable(ObjectPointer scope, std::string const& callableName, bool forceReload) { - __pythonShouldBeInitialized(); + MutexLock::Guard guard(__sLock); + __shouldBeInitialized(); - CallableRef pythonCallable(nullptr); + ObjectPointer pythonCallable(nullptr); - if (!__pythonCallables.count(scopeRef) || !__pythonCallables.at(scopeRef).count(callableName) || forceReload) + if (!__callables.count(scope) || !__callables.at(scope).count(callableName) || forceReload) { - pythonCallable = PyObject_GetAttrString(scopeRef, callableName.c_str()); + pythonCallable = PyObject_GetAttrString(scope, callableName.c_str()); if (!pythonCallable) { - __pythonThrowException("Python::callable(): Cannot access to callable '" + callableName + "'"); + __throwException("Python::callable(): Cannot access to callable '" + callableName + "'"); } if (!PyCallable_Check(pythonCallable)) { - __pythonThrowException("Python::callable(): '" + callableName + "' is not callable"); + __throwException("Python::callable(): '" + callableName + "' is not callable"); } - if (__pythonCallables.count(scopeRef) && __pythonCallables.at(scopeRef).count(callableName)) + if (__callables.count(scope) && __callables.at(scope).count(callableName)) { - __pythonDecRef(callable(scopeRef, callableName, false)); + __decRef(callable(scope, callableName, false)); } - __pythonCallables[scopeRef][callableName] = pythonCallable; + __callables[scope][callableName] = pythonCallable; } else { - pythonCallable = __pythonCallables.at(scopeRef).at(callableName); + pythonCallable = __callables.at(scope).at(callableName); } - __pythonRelease(); - return pythonCallable; } -Python::CallableRef Python::callable(std::string const& moduleName, std::string const& callableName, bool forceReload) +ObjectPointer callable(std::string const& moduleName, std::string const& callableName, bool forceReload) { return callable(module(moduleName), callableName, forceReload); } -Python::ObjectHandler const& Python::arguments(ObjectHandlersList const& args) +ObjectHandler const& call(ObjectPointer callableObject, ObjectHandlersList const& args, bool keepArguments) { - __pythonShouldBeInitialized(); - - const ObjectHandler argsTuple(args.size() ? PyTuple_New(args.size()) : nullptr); + MutexLock::Guard guard(__sLock); + __shouldBeInitialized(); - if (argsTuple) + const auto pyArgsTuple(args.size() ? tuple(args, keepArguments) : nullptr); + const auto pyRet(PyObject_CallObject(callableObject, pyArgsTuple)); + + if (pyArgsTuple) { - __pythonObjects.emplace(argsTuple); - - size_t pos(0); - - for (auto const& object : args) - { - PyTuple_SetItem(argsTuple, pos++, object); - __pythonWithdrawAsCatched(object); - } + forgetArgument(pyArgsTuple); } - else + + if (!pyRet) { - __pythonThrowException("Python::arguments(): failure in creating tuple"); + __throwException("Python::call(): failure in calling callable"); } - __pythonRelease(); - - return *__pythonObjects.find(argsTuple); + return __sObjects.registerObject(pyRet); } -Python::ObjectHandler const& Python::call(CallableRef callableObject, ObjectHandler argumentsObject, bool keepArguments) +ObjectHandler tuple(ObjectHandlersList const& args, bool keepArguments) { - SHLUBLU_TODO("Parameter keepArguments deserves a unit test"); + MutexLock::Guard guard(__sLock); + __shouldBeInitialized(); - __pythonShouldBeInitialized(); + const ObjectHandler tuple(PyTuple_New(args.size())); - if (argumentsObject && !PyTuple_Check(argumentsObject)) + if (!tuple) { - __pythonThrowException("Python::call(): argumentsObject is not a tuple"); + __throwException("Python::tuple(): failure in creating tuple"); } - const auto pyRet(PyObject_CallObject(callableObject, argumentsObject)); - - if (argumentsObject && !keepArguments) - { - forgetArgument(argumentsObject); - } + const auto& ret(__sObjects.registerObject(tuple)); - if (!pyRet) - { - __pythonThrowException("Python::call(): failure in calling callable"); - } + size_t pos(0); - const auto& ret(*__pythonObjects.emplace(pyRet).first); + for (auto const& object : args) + { + __handleObjectUnregistration(object, keepArguments); - __pythonRelease(); + PyTuple_SetItem(tuple, pos++, object); + } return ret; } -Python::ObjectHandler Python::tuple(ObjectHandlersList const& args) +ObjectHandler list(ObjectHandlersList const& args, bool keepArguments) { - __pythonShouldBeInitialized(); + MutexLock::Guard guard(__sLock); + __shouldBeInitialized(); - const ObjectHandler objTuple(PyTuple_New(args.size())); + const ObjectHandler list(PyList_New(args.size())); - size_t pos(0); - - for (auto const& object : args) + if (!list) { - PyTuple_SetItem(objTuple, pos++, object); - __pythonWithdrawAsCatched(object); + __throwException("Python::list(): failure in creating list"); } - __pythonRelease(); - - return objTuple; -} - - -Python::ObjectHandler Python::list(ObjectHandlersList const& args) -{ - __pythonShouldBeInitialized(); - - const ObjectHandler objList(PyList_New(args.size())); + const auto& ret(__sObjects.registerObject(list)); size_t pos(0); for (auto const& object : args) { - PyList_SetItem(objList, pos++, object); - __pythonWithdrawAsCatched(object); - } + __handleObjectUnregistration(object, keepArguments); - __pythonRelease(); + PyList_SetItem(list, pos++, object); + } - return objList; + return ret; } -void Python::addList(ObjectHandler objList, ObjectHandler item, bool keepArguments) +void addList(ObjectHandler objList, ObjectHandler item, bool keepArguments) { - SHLUBLU_TODO("Parameter keepArguments deserves a unit test"); + MutexLock::Guard guard(__sLock); + __shouldBeInitialized(); - __pythonShouldBeInitialized(); - - if (PyList_Check(objList)) - { - PyList_Append(objList, item); - - if (!keepArguments) - { - __pythonDecRef(item); - __pythonWithdrawAsCatched(item); - } - } - else + if (!PyList_Check(objList)) { - __pythonThrowException("Python::addList(): Trying to add an item to an object that is not a list"); + __throwException("Python::addList(): Trying to add an item to an object that is not a list"); } - __pythonRelease(); + __handleObjectUnregistration(item, keepArguments); + + PyList_Append(objList, item); + __decRef(item); } -Python::ObjectHandler Python::fromAscii(std::string const& str) +ObjectHandler fromAscii(std::string const& str) { - __pythonShouldBeInitialized(); + MutexLock::Guard guard(__sLock); + __shouldBeInitialized(); - const auto ret(PyUnicode_FromWideChar(String::toWString(str).c_str(), -1)); + const auto object(PyUnicode_FromWideChar(String::toWString(str).c_str(), -1)); - __pythonRelease(); - - return ret; + return __sObjects.registerObject(object); } -std::string Python::toAscii(ObjectHandler object, bool keepArgument) +std::string toAscii(ObjectHandler object, bool keepArg) { - SHLUBLU_TODO("Parameter keepArguments deserves a unit test"); + MutexLock::Guard guard(__sLock); + __shouldBeInitialized(); - __pythonShouldBeInitialized(); + if (!PyUnicode_Check(object)) + { + __throwException("Python::toAscii(): Trying to convert an object that is not a Unicode string to an ASCII string"); + } - std::string ret; + wchar_t* wstr(PyUnicode_AsWideCharString(object, nullptr)); + const std::string ret(String::fromWString(wstr)); + PyMem_Free(wstr); - if (PyUnicode_Check(object)) - { - wchar_t* wstr(PyUnicode_AsWideCharString(object, nullptr)); - ret = String::fromWString(wstr); - PyMem_Free(wstr); + __handleObjectUnregistration(object, keepArg); - if (!keepArgument) - { - __pythonDecRef(object); - __pythonWithdrawAsCatched(object); - } - } - else + if (!keepArg) { - __pythonThrowException("Python::toAscii(): Trying to convert an object that is not a Unicode string to an ASCII string"); + __decRef(object); } - __pythonRelease(); - return ret; } -Python::ObjectHandler const& Python::keepArgument(ObjectHandler const& object) +ObjectHandler const& keepArgument(ObjectHandler const& object) { - __pythonShouldBeInitialized(); + MutexLock::Guard guard(__sLock); + __shouldBeInitialized(); - if (!__pythonObjects.count(object)) + if (!__sObjects.isRegistered(object)) { - __pythonThrowException("Python::keepArgument(): Argument is not under control"); + __throwException("Python::keepArgument(): object is not under control"); } Py_INCREF(object); const ObjectHandler newHandler(object.get()); - const auto& ret(*__pythonObjects.emplace(newHandler).first); - - __pythonRelease(); - - return ret; + + return __sObjects.registerObject(newHandler); } -Python::ObjectHandler const& Python::controlArgument(ObjectHandler object) +ObjectHandler const& controlArgument(ObjectHandler object) { - __pythonShouldBeInitialized(); + MutexLock::Guard guard(__sLock); + __shouldBeInitialized(); - if (__pythonObjects.count(object)) + if (__sObjects.isRegistered(object)) { - __pythonThrowException("Python::controlArgument(): Trying to give control of an object that is already under control"); + __throwException("Python::controlArgument(): Trying to give control of an object that is already under control"); } - const auto& ret (*__pythonObjects.emplace(object).first); - - __pythonRelease(); - - return ret; + return __sObjects.registerObject(object); } -void Python::forgetArgument(ObjectHandler const& object) +void forgetArgument(ObjectHandler const& object) { - __pythonShouldBeInitialized(); + MutexLock::Guard guard(__sLock); + __shouldBeInitialized(); - if (__pythonObjects.count(object)) + if (!__sObjects.isRegistered(object)) { - __pythonDecRef(object); - __pythonObjects.erase(object); - } - else - { - __pythonThrowException("Python::forgetArgument(): Trying to forget an object that is not under control"); + __throwException("Python::forgetArgument(): Trying to forget an object that is not under control"); } - __pythonRelease(); + __decRef(object); + __sObjects.unregisterObject(object); } } + +} + diff --git a/src/binding/Python_ObjectHandler.cpp b/src/binding/Python_ObjectHandler.cpp index 5ece0b4..8384f03 100644 --- a/src/binding/Python_ObjectHandler.cpp +++ b/src/binding/Python_ObjectHandler.cpp @@ -1,6 +1,3 @@ -#include -SHLUBLU_OPTIMIZE_OFF(); - #include #include @@ -8,15 +5,15 @@ SHLUBLU_OPTIMIZE_OFF(); #include -static shlublu::MutexLock __pythonSequenceLock; - - namespace shlublu { namespace Python { +static MutexLock __sLock; + + int64_t ObjectHandler::Hasher::operator () (ObjectHandler const& key) const { return std::hash()(key.id()); @@ -28,12 +25,10 @@ uint64_t ObjectHandler::sSequence(0); uint64_t ObjectHandler::nextId() { - __pythonSequenceLock.queueLock(); + MutexLock::Guard guard(__sLock); const uint64_t ret(++sSequence); - __pythonSequenceLock.unlock(); - return ret; } diff --git a/src/binding/Python_ObjectHandlersCollection.cpp b/src/binding/Python_ObjectHandlersCollection.cpp new file mode 100644 index 0000000..2301b64 --- /dev/null +++ b/src/binding/Python_ObjectHandlersCollection.cpp @@ -0,0 +1,70 @@ +#include + +#include + + +namespace shlublu +{ + +namespace Python +{ + +ObjectHandlersCollection::ObjectHandlersCollection() + : mObjects() +{} + + +ObjectHandler const& ObjectHandlersCollection::registerObject(ObjectHandler const& oh) +{ + if (mObjects.count(oh)) + { + throw BindingException("ObjectHandlersCollection::registerObject(): object id " + String::xtos(oh.id()) + " is already registered."); + } + + return *(mObjects.emplace(oh).first); +} + + +void ObjectHandlersCollection::unregisterObject(ObjectHandler const& oh) +{ + if (!mObjects.count(oh)) + { + throw BindingException("ObjectHandlersCollection::registerObject(): object id " + String::xtos(oh.id()) + " is not registered."); + } + + mObjects.erase(oh); +} + + +void ObjectHandlersCollection::clear() +{ + mObjects.clear(); +} + + +bool ObjectHandlersCollection::isRegistered(ObjectHandler const& oh) const +{ + return mObjects.count(oh) > 0; +} + + +size_t ObjectHandlersCollection::size() const +{ + return mObjects.size(); +} + + +ObjectHandlersCollection::Storage::const_iterator ObjectHandlersCollection::begin() const +{ + return mObjects.begin(); +} + + +ObjectHandlersCollection::Storage::const_iterator ObjectHandlersCollection::end() const +{ + return mObjects.end(); +} + +} + +} diff --git a/tests-shlublu.vcxproj b/tests-shlublu.vcxproj index 4507bfe..58244cb 100644 --- a/tests-shlublu.vcxproj +++ b/tests-shlublu.vcxproj @@ -175,6 +175,7 @@ + diff --git a/tests-shlublu.vcxproj.filters b/tests-shlublu.vcxproj.filters index cbe62a2..fe91e6a 100644 --- a/tests-shlublu.vcxproj.filters +++ b/tests-shlublu.vcxproj.filters @@ -27,5 +27,8 @@ tests\binding + + tests\binding + \ No newline at end of file diff --git a/tests/binding/TestPython.cpp b/tests/binding/TestPython.cpp index e5aed90..f9a4fcf 100644 --- a/tests/binding/TestPython.cpp +++ b/tests/binding/TestPython.cpp @@ -74,7 +74,7 @@ namespace binding_Python TEST_CLASS(executeTest) { - TEST_METHOD(executeSingleProperHighLevelInstructionWorks) + TEST_METHOD(executeRawCodeWorks) { Python::init("pythonBinding"); Python::execute("a = []"); @@ -83,15 +83,17 @@ namespace binding_Python } - TEST_METHOD(executeSeriesOfProperHighLevelInstructionsWorks) + TEST_METHOD(executeProgramWorks) { Python::init("pythonBinding"); - const Python::Program program = + const Python::Program program { - "a = 5", - "b = 5", - "c = a + b" + { + "a = 5", + "b = 5", + "c = a + b" + } }; Python::execute(program); @@ -100,7 +102,7 @@ namespace binding_Python } - TEST_METHOD(executeSingleWrongHighLevelInstructionThrows) + TEST_METHOD(executeWrongRawCodeThrows) { Python::init("pythonBinding"); @@ -110,15 +112,17 @@ namespace binding_Python } - TEST_METHOD(executeSeriesContainingWrongHighLevelInstructionsThrows) + TEST_METHOD(executeWrongProgramThrows) { Python::init("pythonBinding"); - const Python::Program program = + const Python::Program program { - "a = 5", - "wrong(blahblah)", - "c = a + b" + { + "a = 5", + "wrong(blahblah)", + "c = a + b" + } }; Assert::ExpectException([&program]() { Python::execute(program); }); @@ -132,13 +136,13 @@ namespace binding_Python Python::init("pythonBinding"); Python::shutdown(); - Assert::ExpectException([]() { Python::execute("wrong(blah)"); }); + Assert::ExpectException([]() { Python::execute("a = []"); }); } TEST_METHOD(executeWithoutInitThrows) { - Assert::ExpectException([]() { Python::execute("wrong(blah)"); }); + Assert::ExpectException([]() { Python::execute("a = []"); }); } }; @@ -162,8 +166,8 @@ namespace binding_Python const auto importedA2(Python::import("os")); const auto importedB1(Python::import("sys")); - Assert::AreEqual(static_cast(importedA1), static_cast(importedA2)); - Assert::AreNotEqual(static_cast(importedA1), static_cast(importedB1)); + Assert::IsTrue(importedA1 == importedA2); + Assert::IsFalse(importedA1 == importedB1); Python::shutdown(); } @@ -308,8 +312,8 @@ namespace binding_Python Python::init("pythonBinding"); Python::execute("class TestClass():\n\tdef __init__(self,x):\n\t\tself.xyz=x"); - const auto varA(Python::object(Python::call(Python::callable(Python::moduleMain, "TestClass"), Python::arguments({ PyLong_FromLong(55) })), "xyz")); - const auto varB(Python::object(Python::call(Python::callable(Python::moduleMain, "TestClass"), Python::arguments({ PyLong_FromLong(66) })), "xyz")); + const auto varA(Python::object(Python::call(Python::callable(Python::moduleMain, "TestClass"), { PyLong_FromLong(55) }), "xyz")); + const auto varB(Python::object(Python::call(Python::callable(Python::moduleMain, "TestClass"), { PyLong_FromLong(66) }), "xyz")); Assert::IsNotNull(varA.get()); Assert::AreEqual(55L, PyLong_AsLong(varA)); @@ -373,8 +377,8 @@ namespace binding_Python Python::init("pythonBinding"); Python::execute("class TestClass():\n\tdef __init__(self,x):\n\t\tself.xyz=x\n\n\tdef ret(self):\n\t\treturn self.xyz"); - const auto varA(Python::call(Python::callable(Python::moduleMain, "TestClass"), Python::arguments({ PyLong_FromLong(55) }))); - const auto varB(Python::call(Python::callable(Python::moduleMain, "TestClass"), Python::arguments({ PyLong_FromLong(66) }))); + const auto varA(Python::call(Python::callable(Python::moduleMain, "TestClass"), { PyLong_FromLong(55) })); + const auto varB(Python::call(Python::callable(Python::moduleMain, "TestClass"), { PyLong_FromLong(66) })); Assert::IsNotNull(varA.get()); Assert::AreEqual(55L, PyLong_AsLong(Python::call(Python::callable(varA, "ret")))); @@ -461,31 +465,6 @@ namespace binding_Python }; - TEST_CLASS(argumentsTest) - { - TEST_METHOD(argumentsWorks) - { - Python::init("pythonBinding"); - - const auto args(Python::arguments({ PyLong_FromLong(5), PyFloat_FromDouble(0.42), Python::fromAscii("Test") })); - - Assert::IsTrue(PyTuple_CheckExact(args)); - - Assert::AreEqual(Py_ssize_t(3), PyTuple_Size(args)); - - Assert::IsTrue(PyLong_CheckExact(PyTuple_GetItem(args, 0))); - Assert::IsTrue(PyFloat_CheckExact(PyTuple_GetItem(args, 1))); - Assert::IsTrue(PyUnicode_CheckExact(PyTuple_GetItem(args, 2))); - - Assert::AreEqual(5L, PyLong_AsLong(PyTuple_GetItem(args, 0))); - Assert::AreEqual(0.42, PyFloat_AsDouble(PyTuple_GetItem(args, 1))); - Assert::AreEqual(std::string("Test"), Python::toAscii(PyTuple_GetItem(args, 2), true)); - - Python::shutdown(); - } - }; - - TEST_CLASS(callTest) { TEST_METHOD(callWorksWithMainModule) @@ -501,7 +480,7 @@ namespace binding_Python Python::call ( Python::callable(Python::moduleMain, "sumTest"), - Python::arguments({ PyLong_FromLong(1), PyFloat_FromDouble(2.2) }) + { PyLong_FromLong(1), PyFloat_FromDouble(2.2) } ) ) ); @@ -523,7 +502,7 @@ namespace binding_Python Python::call ( Python::callable(Python::moduleBuiltins, "str"), - Python::arguments({ PyFloat_FromDouble(2.2) }) + { PyFloat_FromDouble(2.2) } ) ) ); @@ -538,8 +517,8 @@ namespace binding_Python { Python::init("pythonBinding"); - Python::execute("class TestClass():\n\tdef __init__(self,x):\n\t\tself.xyz=x\n\n\tdef ret(self):\n\t\treturn self.xyz"); - const auto var(Python::call(Python::callable(Python::moduleMain, "TestClass"), Python::arguments({ PyLong_FromLong(55) }))); + Python::execute({ "class TestClass():", "\tdef __init__(self,x):", "\t\tself.xyz=x", "\tdef ret(self):", "\t\treturn self.xyz" }); + const auto var(Python::call(Python::callable(Python::moduleMain, "TestClass"), { PyLong_FromLong(55) })); Assert::AreEqual(55L, PyLong_AsLong(Python::call(Python::callable(var, "ret")))); @@ -547,55 +526,11 @@ namespace binding_Python } - TEST_METHOD(callWorksWithCustomTuple) - { - Python::init("pythonBinding"); - - Python::execute("def sumTest(a, b):\n\treturn a + b"); - - const auto resultSum - ( - PyFloat_AsDouble - ( - Python::call - ( - Python::callable(Python::moduleMain, "sumTest"), - Python::controlArgument(Python::tuple({ PyLong_FromLong(1), PyFloat_FromDouble(2.2) })) - ) - ) - ); - - Assert::AreEqual(3.2, resultSum); - - Python::shutdown(); - } - - - TEST_METHOD(callThrowsWithNonTupleArguments) - { - Python::init("pythonBinding"); - - Assert::ExpectException - ( - []() - { - Python::call - ( - Python::callable(Python::moduleBuiltins, "print"), - Python::controlArgument(Python::fromAscii("test")) - ); - } - ); - - Python::shutdown(); - } - - TEST_METHOD(callThrowsAfterShutdown) { Python::init("pythonBinding"); - Python::execute("def sumTest(a, b):\n\treturn a + b"); + Python::execute(Python::Program{ "def sumTest(a, b):", "\treturn a + b" }); Python::shutdown(); @@ -610,13 +545,51 @@ namespace binding_Python Python::call ( Python::callable(Python::moduleMain, "sumTest"), - Python::arguments({ PyLong_FromLong(1), PyFloat_FromDouble(2.2) }) + { PyLong_FromLong(1), PyFloat_FromDouble(2.2) } ) ) ); } ); } + + + TEST_METHOD(callKeepsCPythonArgumentIfRequested) + { + Python::init("pythonBinding"); + + const auto arg(PyLong_FromLong(42)); + const auto refsCount(arg->ob_refcnt); + + Python::call(Python::callable(Python::moduleBuiltins, "str"), { arg }, true); + + Assert::AreEqual(refsCount, arg->ob_refcnt); + + Python::call(Python::callable(Python::moduleBuiltins, "str"), { arg }, false); + + Assert::AreEqual(refsCount - 1, arg->ob_refcnt); + + Python::shutdown(); + } + + + TEST_METHOD(callKeepsControlledArgumentIfRequested) + { + Python::init("pythonBinding"); + + const auto arg(Python::controlArgument(PyLong_FromLong(42))); + const auto refsCount(arg.get()->ob_refcnt); + + Python::call(Python::callable(Python::moduleBuiltins, "str"), { arg }, true); + + Assert::AreEqual(refsCount, arg.get()->ob_refcnt); + + Python::call(Python::callable(Python::moduleBuiltins, "str"), { arg }, false); + + Assert::AreEqual(refsCount - 1, arg.get()->ob_refcnt); + + Python::shutdown(); + } }; @@ -642,6 +615,60 @@ namespace binding_Python Python::shutdown(); } + + + TEST_METHOD(tupleKeepsCPythonArgumentIfRequested) + { + Python::init("pythonBinding"); + + const auto arg(PyLong_FromLong(42)); + const auto refsCount(arg->ob_refcnt); + + auto tuple(Python::tuple({ arg }, true)); + + Assert::AreEqual(refsCount + 1, arg->ob_refcnt); + + Python::forgetArgument(tuple); + + Assert::AreEqual(refsCount, arg->ob_refcnt); + + tuple = Python::tuple({ arg }, false); + + Assert::AreEqual(refsCount, arg->ob_refcnt); + + Python::forgetArgument(tuple); + + Assert::AreEqual(refsCount - 1, arg->ob_refcnt); + + Python::shutdown(); + } + + + TEST_METHOD(tupleKeepsControlledArgumentIfRequested) + { + Python::init("pythonBinding"); + + const auto arg(Python::controlArgument(PyLong_FromLong(42))); + const auto refsCount(arg.get()->ob_refcnt); + + auto tuple(Python::tuple({ arg }, true)); + + Assert::AreEqual(refsCount + 1, arg.get()->ob_refcnt); + + Python::forgetArgument(tuple); + + Assert::AreEqual(refsCount, arg.get()->ob_refcnt); + + tuple = Python::tuple({ arg }, false); + + Assert::AreEqual(refsCount, arg.get()->ob_refcnt); + + Python::forgetArgument(tuple); + + Assert::AreEqual(refsCount - 1, arg.get()->ob_refcnt); + + Python::shutdown(); + } }; @@ -690,6 +717,118 @@ namespace binding_Python Python::shutdown(); } + + + TEST_METHOD(listKeepsCPythonArgumentIfRequested) + { + Python::init("pythonBinding"); + + const auto arg(PyLong_FromLong(42)); + const auto refsCount(arg->ob_refcnt); + + auto list(Python::list({ arg }, true)); + + Assert::AreEqual(refsCount + 1, arg->ob_refcnt); + + Python::forgetArgument(list); + + Assert::AreEqual(refsCount, arg->ob_refcnt); + + list = Python::list({ arg }, false); + + Assert::AreEqual(refsCount, arg->ob_refcnt); + + Python::forgetArgument(list); + + Assert::AreEqual(refsCount - 1, arg->ob_refcnt); + + Python::shutdown(); + } + + + TEST_METHOD(listKeepsControlledArgumentIfRequested) + { + Python::init("pythonBinding"); + + const auto arg(Python::controlArgument(PyLong_FromLong(42))); + const auto refsCount(arg.get()->ob_refcnt); + + auto list(Python::list({ arg }, true)); + + Assert::AreEqual(refsCount + 1, arg.get()->ob_refcnt); + + Python::forgetArgument(list); + + Assert::AreEqual(refsCount, arg.get()->ob_refcnt); + + list = Python::list({ arg }, false); + + Assert::AreEqual(refsCount, arg.get()->ob_refcnt); + + Python::forgetArgument(list); + + Assert::AreEqual(refsCount - 1, arg.get()->ob_refcnt); + + Python::shutdown(); + } + + + TEST_METHOD(addListKeepsCPythonArgumentIfRequested) + { + Python::init("pythonBinding"); + + const auto arg(PyLong_FromLong(42)); + const auto refsCount(arg->ob_refcnt); + + auto list(Python::list()); + Python::addList(list, arg, true); + + Assert::AreEqual(refsCount + 1, arg->ob_refcnt); + + Python::forgetArgument(list); + + Assert::AreEqual(refsCount, arg->ob_refcnt); + + list = Python::list(); + Python::addList(list, arg, false); + + Assert::AreEqual(refsCount, arg->ob_refcnt); + + Python::forgetArgument(list); + + Assert::AreEqual(refsCount - 1, arg->ob_refcnt); + + Python::shutdown(); + } + + + TEST_METHOD(addListKeepsControlledArgumentIfRequested) + { + Python::init("pythonBinding"); + + const auto arg(Python::controlArgument(PyLong_FromLong(42))); + const auto refsCount(arg.get()->ob_refcnt); + + auto list(Python::list()); + Python::addList(list, arg, true); + + Assert::AreEqual(refsCount + 1, arg.get()->ob_refcnt); + + Python::forgetArgument(list); + + Assert::AreEqual(refsCount, arg.get()->ob_refcnt); + + list = Python::list(); + Python::addList(list, arg, false); + + Assert::AreEqual(refsCount, arg.get()->ob_refcnt); + + Python::forgetArgument(list); + + Assert::AreEqual(refsCount - 1, arg.get()->ob_refcnt); + + Python::shutdown(); + } }; @@ -704,8 +843,8 @@ namespace binding_Python const auto ok(Python::fromAscii("testString")); const auto ko(Python::fromAscii("otherString")); - Assert::IsTrue(Python::call(Python::callable(Python::moduleMain, "checkEqual"), Python::arguments({ ok })).get() == Py_True); - Assert::IsTrue(Python::call(Python::callable(Python::moduleMain, "checkEqual"), Python::arguments({ ko })).get() == Py_False); + Assert::IsTrue(Python::call(Python::callable(Python::moduleMain, "checkEqual"), { ok }).get() == Py_True); + Assert::IsTrue(Python::call(Python::callable(Python::moduleMain, "checkEqual"), { ko }).get() == Py_False); Python::shutdown(); } @@ -722,6 +861,27 @@ namespace binding_Python Python::shutdown(); } + + + TEST_METHOD(toAsciiKeepsArgumentIfReqested) + { + Python::init("pythonBinding"); + + Python::execute(Python::Program { "def retString():", "\treturn 'string'" }); + + const auto str(Python::call(Python::callable(Python::moduleMain, "retString"))); + const auto refsCount(str.get()->ob_refcnt); + + auto ascii(Python::toAscii(str, true)); + + Assert::AreEqual(refsCount + 1, str.get()->ob_refcnt); + + ascii = Python::toAscii(str, false); + + Assert::AreEqual(refsCount, str.get()->ob_refcnt); + + Python::shutdown(); + } }; TEST_CLASS(referencesCountTest) @@ -777,22 +937,17 @@ namespace binding_Python { Python::init("pythonBinding"); - Python::execute("def returnX():\n\treturn 5"); - - Assert::ExpectException([]() { Python::keepArgument(Python::fromAscii("test")); }); + Assert::ExpectException([]() { Python::keepArgument(PyLong_FromLong(5)); }); Python::shutdown(); } + TEST_METHOD(controlArgumentThrowsIfAlreadyUnderControl) { Python::init("pythonBinding"); - Python::execute("def returnX():\n\treturn 5"); - - const auto val(Python::call(Python::callable(Python::moduleMain, "returnX"))); - - Assert::ExpectException([&val]() { Python::controlArgument(val); }); + Assert::ExpectException([]() { Python::controlArgument(Python::fromAscii("test")); }); Python::shutdown(); } @@ -836,12 +991,6 @@ namespace binding_Python { Python::init("pythonBinding"); - Python::execute(Python::Program({ "def inc(x):", "\treturn x + 1" })); - - const long nbIt(2000000L); - long x(0); - long y(0); - const auto job ( [](long iterations) -> long @@ -851,7 +1000,11 @@ namespace binding_Python for (long i = 0; i < iterations; ++i) { - const auto pyVal(Python::call(inc, Python::arguments({ PyLong_FromLong(v) }))); + Python::beginCriticalSection(); + const auto arg(PyLong_FromLong(v)); + Python::endCriticalSection(); + + const auto pyVal(Python::call(inc, { arg })); v = PyLong_AsLong(pyVal); @@ -862,6 +1015,12 @@ namespace binding_Python } ); + Python::execute(Python::Program({ "def inc(x):", "\treturn x + 1" })); + + const long nbIt(1000000L); + long x(0); + long y(0); + std::thread tx ( [&x, &job, nbIt]() @@ -885,7 +1044,7 @@ namespace binding_Python { Python::init("pythonBinding"); - const long nbIt(25000L); + const long nbIt(15000L); std::vector values; std::thread producer @@ -899,7 +1058,7 @@ namespace binding_Python { Python::beginCriticalSection(); - const auto res(Python::call(dbl, Python::arguments({ PyLong_FromLong(i) }))); + const auto res(Python::call(dbl, { PyLong_FromLong(i) })); Assert::AreEqual(2 * i, PyLong_AsLong(res)); values.push_back(res); diff --git a/tests/binding/TestPython_ObjectHandlersColection.cpp b/tests/binding/TestPython_ObjectHandlersColection.cpp new file mode 100644 index 0000000..368132c --- /dev/null +++ b/tests/binding/TestPython_ObjectHandlersColection.cpp @@ -0,0 +1,162 @@ +#define NOMINMAX + +#include "CppUnitTest.h" + +#include +#include +SHLUBLU_OPTIMIZE_OFF(); + + +using namespace Microsoft::VisualStudio::CppUnitTestFramework; +using namespace shlublu; + +namespace binding_Python_ObjectHandlersCollection +{ + TEST_CLASS(oebjctHandlersCollectionTest) + { + TEST_METHOD(emptyConstructorWorks) + { + Python::ObjectHandlersCollection c; + + Assert::AreEqual(size_t(0), c.size()); + } + + + TEST_METHOD(registerObjectAddsUnknownObjectsProperly) + { + Python::ObjectHandlersCollection c; + + const Python::ObjectHandler oh(reinterpret_cast(42)); + + Assert::AreEqual(size_t(0), c.size()); + Assert::IsFalse(c.isRegistered(oh)); + + const auto res(c.registerObject(oh)); + + Assert::AreEqual(res.id(), oh.id()); + Assert::IsTrue(res.get() == oh.get()); + + c.registerObject(Python::ObjectHandler(oh.get())); + + Assert::AreEqual(size_t(2), c.size()); + Assert::IsTrue(c.isRegistered(oh)); + } + + + TEST_METHOD(registerObjectThrowsWhenAttemptingToAddKnownObjects) + { + Python::ObjectHandlersCollection c; + + const Python::ObjectHandler oh(reinterpret_cast(42)); + + c.registerObject(oh); + + Assert::ExpectException([&c, &oh]() { c.registerObject(oh); }); + } + + + TEST_METHOD(unregisterObjectRemovesKnownObjectsProperly) + { + Python::ObjectHandlersCollection c; + + const Python::ObjectHandler oh(reinterpret_cast(42)); + + c.registerObject(oh); + c.registerObject(Python::ObjectHandler(oh.get())); + + Assert::AreEqual(size_t(2), c.size()); + Assert::IsTrue(c.isRegistered(oh)); + + c.unregisterObject(oh); + + Assert::AreEqual(size_t(1), c.size()); + Assert::IsFalse(c.isRegistered(oh)); + } + + + TEST_METHOD(unregisterObjectThrowsWhenAttemptingToRemoveUnknownObjects) + { + Python::ObjectHandlersCollection c; + + const Python::ObjectHandler oh(reinterpret_cast(42)); + + c.registerObject(oh); + + Assert::ExpectException([&c, &oh]() { c.unregisterObject(Python::ObjectHandler(oh.get())); }); + } + + + TEST_METHOD(clearEmptiesEverything) + { + Python::ObjectHandlersCollection c; + + const Python::ObjectHandler oh(reinterpret_cast(42)); + + c.registerObject(oh); + c.registerObject(oh.get()); + + Assert::AreEqual(size_t(2), c.size()); + + c.clear(); + + Assert::AreEqual(size_t(0), c.size()); + } + + + TEST_METHOD(isRegisteredReturnsTrueForRegisteredObjects) + { + Python::ObjectHandlersCollection c; + + const Python::ObjectHandler oh(reinterpret_cast(42)); + + Assert::IsFalse(c.isRegistered(oh)); + + c.registerObject(oh); + + Assert::IsTrue(c.isRegistered(oh)); + } + + + TEST_METHOD(isRegisteredReturnsFalseForNonRegisteredObjects) + { + Python::ObjectHandlersCollection c; + + const Python::ObjectHandler oh(reinterpret_cast(42)); + + c.registerObject(oh); + + Assert::IsTrue(c.isRegistered(oh)); + + c.unregisterObject(oh); + + Assert::IsFalse(c.isRegistered(oh)); + } + + + TEST_METHOD(sizeCorrespondstoTheNumberOfObects) + { + Python::ObjectHandlersCollection c; + + const Python::ObjectHandler oh(reinterpret_cast(42)); + + Assert::AreEqual(size_t(0), c.size()); + + c.registerObject(oh); + + Assert::AreEqual(size_t(1), c.size()); + + c.registerObject(Python::ObjectHandler(oh.get())); + + Assert::AreEqual(size_t(2), c.size()); + }; + + + TEST_METHOD(beginsAndEndAreEqualWhenEmpty) + { + Python::ObjectHandlersCollection c; + + Assert::IsTrue(c.begin() == c.end()); + } + }; +} +